You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2012/12/03 23:13:41 UTC
git commit: [#5265] Control Solr commit options from config file
Updated Branches:
refs/heads/master 8c8ad4dde -> 9e08c7d0a
[#5265] Control Solr commit options from config file
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/9e08c7d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/9e08c7d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/9e08c7d0
Branch: refs/heads/master
Commit: 9e08c7d0abfe79dc01ed3a0efe992f2114fc6efd
Parents: 8c8ad4d
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Nov 20 21:12:27 2012 +0000
Committer: Cory Johns <jo...@geek.net>
Committed: Mon Dec 3 22:13:24 2012 +0000
----------------------------------------------------------------------
Allura/allura/lib/app_globals.py | 73 ++--------------------
Allura/allura/lib/solr.py | 91 ++++++++++++++++++++++++++++
Allura/allura/tests/unit/test_solr.py | 30 +++++++++
Allura/development.ini | 4 +
4 files changed, 131 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 1789855..a263177 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -6,14 +6,12 @@ __all__ = ['Globals']
import logging
import cgi
import json
-import shlex
import datetime
from urllib import urlencode
from subprocess import Popen, PIPE
import activitystream
import pkg_resources
-import pysolr
import markdown
import pygments
import pygments.lexers
@@ -38,21 +36,24 @@ from allura.lib import helpers as h
from allura.lib.widgets import analytics
from allura.lib.security import Credentials
from allura.lib.async import Connection, MockAMQ
+from allura.lib.solr import Solr, MockSOLR
from allura.lib.zarkov_helpers import ZarkovClient, zmq
log = logging.getLogger(__name__)
+
class ForgeMarkdown(markdown.Markdown):
def convert(self, source):
try:
return markdown.Markdown.convert(self, source)
- except Exception as e:
+ except Exception:
log.info('Invalid markdown: %s', source, exc_info=True)
escaped = h.really_unicode(source)
escaped = cgi.escape(escaped)
return h.html.literal(u"""<p><strong>ERROR!</strong> The markdown supplied could not be parsed correctly.
Did you forget to surround a code snippet with "~~~~"?</p><pre>%s</pre>""" % escaped)
+
class Globals(object):
"""Container for objects available throughout the life of the application.
@@ -72,7 +73,8 @@ class Globals(object):
if asbool(config.get('solr.mock')):
self.solr = MockSOLR()
elif self.solr_server:
- self.solr = pysolr.Solr(self.solr_server)
+ self.solr = Solr(self.solr_server, commit=asbool(config.get('solr.commit', True)),
+ commitWithin=config.get('solr.commitWithin'))
else: # pragma no cover
self.solr = None
self.use_queue = asbool(config.get('use_queue', False))
@@ -422,69 +424,6 @@ class Globals(object):
def year(self):
return datetime.datetime.utcnow().year
-class MockSOLR(object):
-
- class MockHits(list):
- @property
- def hits(self):
- return len(self)
-
- @property
- def docs(self):
- return self
-
- def __init__(self):
- self.db = {}
-
- def add(self, objects):
- for o in objects:
- o['text'] = ''.join(o['text'])
- self.db[o['id']] = o
-
- def commit(self):
- pass
-
- def search(self, q, fq=None, **kw):
- if isinstance(q, unicode):
- q = q.encode('latin-1')
- # Parse query
- preds = []
- q_parts = shlex.split(q)
- if fq: q_parts += fq
- for part in q_parts:
- if part == '&&':
- continue
- if ':' in part:
- field, value = part.split(':', 1)
- preds.append((field, value))
- else:
- preds.append(('text', part))
- result = self.MockHits()
- for obj in self.db.values():
- for field, value in preds:
- neg = False
- if field[0] == '!':
- neg = True
- field = field[1:]
- if field == 'text' or field.endswith('_t'):
- if (value not in str(obj.get(field, ''))) ^ neg:
- break
- else:
- if (value != str(obj.get(field, ''))) ^ neg:
- break
- else:
- result.append(obj)
- return result
-
- def delete(self, *args, **kwargs):
- if kwargs.get('q', None) == '*:*':
- self.db = {}
- elif kwargs.get('id', None):
- del self.db[kwargs['id']]
- elif kwargs.get('q', None):
- for doc in self.search(kwargs['q']):
- self.delete(id=doc['id'])
-
class Icon(object):
def __init__(self, char, css):
self.char = char
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
new file mode 100644
index 0000000..99af9f1
--- /dev/null
+++ b/Allura/allura/lib/solr.py
@@ -0,0 +1,91 @@
+import shlex
+import pysolr
+
+
+class Solr(pysolr.Solr):
+ """Solr server that accepts default values for `commit` and
+ `commitWithin` and passes those values through to each `add` and
+ `delete` call, unless explicitly overridden.
+ """
+
+ def __init__(self, server, commit=True, commitWithin=None):
+ pysolr.Solr.__init__(self, server)
+ self.commit = commit
+ self.commitWithin = commitWithin
+
+ def add(self, *args, **kw):
+ if 'commit' not in kw:
+ kw['commit'] = self.commit
+ if self.commitWithin and 'commitWithin' not in kw:
+ kw['commitWithin'] = self.commitWithin
+ return pysolr.Solr.add(self, *args, **kw)
+
+ def delete(self, *args, **kw):
+ if 'commit' not in kw:
+ kw['commit'] = self.commit
+ return pysolr.Solr.delete(self, *args, **kw)
+
+
+class MockSOLR(object):
+
+ class MockHits(list):
+ @property
+ def hits(self):
+ return len(self)
+
+ @property
+ def docs(self):
+ return self
+
+ def __init__(self):
+ self.db = {}
+
+ def add(self, objects):
+ for o in objects:
+ o['text'] = ''.join(o['text'])
+ self.db[o['id']] = o
+
+ def commit(self):
+ pass
+
+ def search(self, q, fq=None, **kw):
+ if isinstance(q, unicode):
+ q = q.encode('latin-1')
+ # Parse query
+ preds = []
+ q_parts = shlex.split(q)
+ if fq: q_parts += fq
+ for part in q_parts:
+ if part == '&&':
+ continue
+ if ':' in part:
+ field, value = part.split(':', 1)
+ preds.append((field, value))
+ else:
+ preds.append(('text', part))
+ result = self.MockHits()
+ for obj in self.db.values():
+ for field, value in preds:
+ neg = False
+ if field[0] == '!':
+ neg = True
+ field = field[1:]
+ if field == 'text' or field.endswith('_t'):
+ if (value not in str(obj.get(field, ''))) ^ neg:
+ break
+ else:
+ if (value != str(obj.get(field, ''))) ^ neg:
+ break
+ else:
+ result.append(obj)
+ return result
+
+ def delete(self, *args, **kwargs):
+ if kwargs.get('q', None) == '*:*':
+ self.db = {}
+ elif kwargs.get('id', None):
+ del self.db[kwargs['id']]
+ elif kwargs.get('q', None):
+ for doc in self.search(kwargs['q']):
+ self.delete(id=doc['id'])
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/tests/unit/test_solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
new file mode 100644
index 0000000..2e5b668
--- /dev/null
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -0,0 +1,30 @@
+import unittest
+import mock
+
+from allura.lib.solr import Solr
+
+class TestSolr(unittest.TestCase):
+ @mock.patch('allura.lib.solr.pysolr')
+ def setUp(self, pysolr):
+ self.solr = Solr('server', commit=False, commitWithin='10000')
+
+ @mock.patch('allura.lib.solr.pysolr')
+ def test_add(self, pysolr):
+ s = self.solr
+ s.add('foo', commit=True, commitWithin=None)
+ pysolr.Solr.add.assert_called_once_with(s, 'foo', commit=True,
+ commitWithin=None)
+ pysolr.reset_mock()
+ s.add('bar', somekw='value')
+ pysolr.Solr.add.assert_called_once_with(s, 'bar', commit=False,
+ commitWithin='10000', somekw='value')
+
+ @mock.patch('allura.lib.solr.pysolr')
+ def test_delete(self, pysolr):
+ s = self.solr
+ s.delete('foo', commit=True)
+ pysolr.Solr.delete.assert_called_once_with(s, 'foo', commit=True)
+ pysolr.reset_mock()
+ s.delete('bar', somekw='value')
+ pysolr.Solr.delete.assert_called_once_with(s, 'bar', commit=False,
+ somekw='value')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 4ba35d7..7ac4897 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -157,6 +157,10 @@ openid.realm = http://localhost:8080/
# SOLR setup
solr.server = http://localhost:8983/solr/core0
+# commit on every add/delete?
+solr.commit = false
+# commit add operations within N ms
+solr.commitWithin = 10000
# Forgemail server
forgemail.host = 0.0.0.0