You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2013/05/08 00:29:55 UTC
[25/50] [abbrv] git commit: [#6161] ticket:329 Push changes to
multiple solr instances
[#6161] ticket:329 Push changes to multiple solr instances
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/7a332c6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/7a332c6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/7a332c6e
Branch: refs/heads/tv/2053
Commit: 7a332c6ecdc6c2cffd5f262d0293d7afdf195d85
Parents: ba100cc
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Apr 29 12:46:33 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu May 2 22:18:14 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/app_globals.py | 19 +++++---
Allura/allura/lib/solr.py | 47 +++++++++++++++-----
Allura/allura/tests/unit/test_solr.py | 67 +++++++++++++++++++++------
Allura/development.ini | 3 +
4 files changed, 104 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0621b8d..b030d6f 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -38,7 +38,7 @@ import pygments.util
from tg import config, session
from pylons import request
from pylons import tmpl_context as c
-from paste.deploy.converters import asbool, asint
+from paste.deploy.converters import asbool, asint, aslist
from pypeline.markup import markup as pypeline_markup
import ew as ew_core
@@ -93,14 +93,21 @@ class Globals(object):
self.allura_templates = pkg_resources.resource_filename('allura', 'templates')
# Setup SOLR
- self.solr_server = config.get('solr.server')
+ self.solr_server = aslist(config.get('solr.server'), ',')
+ # skip empty strings in case of extra commas
+ self.solr_server = [s for s in self.solr_server if s]
+ self.solr_query_server = config.get('solr.query_server')
if asbool(config.get('solr.mock')):
self.solr = self.solr_short_timeout = MockSOLR()
elif self.solr_server:
- self.solr = Solr(self.solr_server, commit=asbool(config.get('solr.commit', True)),
- commitWithin=config.get('solr.commitWithin'), timeout=int(config.get('solr.long_timeout', 60)))
- self.solr_short_timeout = Solr(self.solr_server, commit=asbool(config.get('solr.commit', True)),
- commitWithin=config.get('solr.commitWithin'), timeout=int(config.get('solr.short_timeout', 10)))
+ self.solr = Solr(self.solr_server, self.solr_query_server,
+ commit=asbool(config.get('solr.commit', True)),
+ commitWithin=config.get('solr.commitWithin'),
+ timeout=int(config.get('solr.long_timeout', 60)))
+ self.solr_short_timeout = Solr(self.solr_server, self.solr_query_server,
+ commit=asbool(config.get('solr.commit', True)),
+ commitWithin=config.get('solr.commitWithin'),
+ timeout=int(config.get('solr.short_timeout', 10)))
else: # pragma no cover
self.solr = None
self.solr_short_timeout = None
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index e91b183..e1e9567 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -20,28 +20,53 @@ import pysolr
from pysolr import SolrError
-class Solr(pysolr.Solr):
- """Solr server that accepts default values for `commit` and
- `commitWithin` and passes those values through to each `add` and
- `delete` call, unless explicitly overridden.
+class Solr(object):
+ """Solr interface that pushes updates to multiple solr instances.
+
+ `push_servers`: list of servers to push to.
+ `query_server`: server to read from. Uses `push_servers[0]` if not specified.
+
+ Also, accepts default values for `commit` and `commitWithin`
+ and passes those values through to each `add` and `delete` call,
+ unless explicitly overridden.
"""
- def __init__(self, server, commit=True, commitWithin=None, **kw):
- pysolr.Solr.__init__(self, server, **kw)
- self.commit = commit
+ def __init__(self, push_servers, query_server=None,
+ commit=True, commitWithin=None, **kw):
+ self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
+ if query_server:
+ self.query_server = pysolr.Solr(query_server, **kw)
+ else:
+ self.query_server = self.push_pool[0]
+ self._commit = commit
self.commitWithin = commitWithin
def add(self, *args, **kw):
if 'commit' not in kw:
- kw['commit'] = self.commit
+ kw['commit'] = self._commit
if self.commitWithin and 'commitWithin' not in kw:
kw['commitWithin'] = self.commitWithin
- return pysolr.Solr.add(self, *args, **kw)
+ responses = []
+ for solr in self.push_pool:
+ responses.append(solr.add(*args, **kw))
+ return responses
def delete(self, *args, **kw):
if 'commit' not in kw:
- kw['commit'] = self.commit
- return pysolr.Solr.delete(self, *args, **kw)
+ kw['commit'] = self._commit
+ responses = []
+ for solr in self.push_pool:
+ responses.append(solr.delete(*args, **kw))
+ return responses
+
+ def commit(self, *args, **kw):
+ responses = []
+ for solr in self.push_pool:
+ responses.append(solr.commit(*args, **kw))
+ return responses
+
+ def search(self, *args, **kw):
+ return self.query_server.search(*args, **kw)
class MockSOLR(object):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/tests/unit/test_solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index b3dee43..2bc5cdb 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -28,30 +28,67 @@ from allura.lib.solr import Solr
from allura.lib.search import solarize, search_app
class TestSolr(unittest.TestCase):
+
@mock.patch('allura.lib.solr.pysolr')
- def setUp(self, pysolr):
- self.solr = Solr('server', commit=False, commitWithin='10000')
+ def test_init(self, pysolr):
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ calls = [mock.call('server1'), mock.call('server2')]
+ pysolr.Solr.assert_has_calls(calls)
+ assert_equal(len(solr.push_pool), 2)
+
+ pysolr.reset_mock()
+ solr = Solr(servers, 'server3', commit=False, commitWithin='10000')
+ calls = [mock.call('server1'), mock.call('server2'), mock.call('server3')]
+ pysolr.Solr.assert_has_calls(calls)
+ assert_equal(len(solr.push_pool), 2)
@mock.patch('allura.lib.solr.pysolr')
def test_add(self, pysolr):
- s = self.solr
- s.add('foo', commit=True, commitWithin=None)
- pysolr.Solr.add.assert_called_once_with(s, 'foo', commit=True,
- commitWithin=None)
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ solr.add('foo', commit=True, commitWithin=None)
+ calls = [mock.call('foo', commit=True, commitWithin=None)] * 2
+ pysolr.Solr().add.assert_has_calls(calls)
pysolr.reset_mock()
- s.add('bar', somekw='value')
- pysolr.Solr.add.assert_called_once_with(s, 'bar', commit=False,
- commitWithin='10000', somekw='value')
+ solr.add('bar', somekw='value')
+ calls = [mock.call('bar', commit=False,
+ commitWithin='10000', somekw='value')] * 2
+ pysolr.Solr().add.assert_has_calls(calls)
@mock.patch('allura.lib.solr.pysolr')
def test_delete(self, pysolr):
- s = self.solr
- s.delete('foo', commit=True)
- pysolr.Solr.delete.assert_called_once_with(s, 'foo', commit=True)
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ solr.delete('foo', commit=True)
+ calls = [mock.call('foo', commit=True)] * 2
+ pysolr.Solr().delete.assert_has_calls(calls)
+ pysolr.reset_mock()
+ solr.delete('bar', somekw='value')
+ calls = [mock.call('bar', commit=False, somekw='value')] * 2
+ pysolr.Solr().delete.assert_has_calls(calls)
+
+ @mock.patch('allura.lib.solr.pysolr')
+ def test_commit(self, pysolr):
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ solr.commit('arg')
+ pysolr.Solr().commit.assert_has_calls([mock.call('arg')] * 2)
pysolr.reset_mock()
- s.delete('bar', somekw='value')
- pysolr.Solr.delete.assert_called_once_with(s, 'bar', commit=False,
- somekw='value')
+ solr.commit('arg', kw='kw')
+ calls = [mock.call('arg', kw='kw')] * 2
+ pysolr.Solr().commit.assert_has_calls(calls)
+
+ @mock.patch('allura.lib.solr.pysolr')
+ def test_search(self, pysolr):
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ solr.search('foo')
+ solr.query_server.search.assert_called_once_with('foo')
+ pysolr.reset_mock()
+ solr.search('bar', kw='kw')
+ solr.query_server.search.assert_called_once_with('bar', kw='kw')
+
class TestSolarize(unittest.TestCase):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 761876b..b2ac7bf 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -177,7 +177,10 @@ amqp.enabled = false
openid.realm = http://localhost:8080/
# SOLR setup
+# comma-separated list of servers to push changes to
solr.server = http://localhost:8983/solr/core0
+# server to read from. Uses solr.server[0] if not specified.
+# solr.query_server =
# commit on every add/delete?
solr.commit = false
# commit add operations within N ms