You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bloodhound.apache.org by ju...@apache.org on 2013/01/15 10:32:54 UTC
svn commit: r1433322 [1/2] - in
/incubator/bloodhound/branches/bep_0003_multiproduct: ./
bloodhound_dashboard/ bloodhound_multiproduct/ bloodhound_search/
bloodhound_search/bhsearch/ bloodhound_search/bhsearch/templates/
bloodhound_search/bhsearch/test...
Author: jure
Date: Tue Jan 15 09:32:53 2013
New Revision: 1433322
URL: http://svn.apache.org/viewvc?rev=1433322&view=rev
Log:
Sync merge from trunk
Added:
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/wiki_search.py
- copied unchanged from r1433320, incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/wiki_search.py
- copied unchanged from r1433320, incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py
Modified:
incubator/bloodhound/branches/bep_0003_multiproduct/ (props changed)
incubator/bloodhound/branches/bep_0003_multiproduct/NOTICE
incubator/bloodhound/branches/bep_0003_multiproduct/RELEASE_NOTES
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_dashboard/setup.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/setup.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/api.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/query_parser.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/templates/bhsearch.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/__init__.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/api.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/index_with_whoosh.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/ticket_search.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/utils.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/web_ui.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/whoosh_backend.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/ticket_search.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/web_ui.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/whoosh_backend.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/setup.py
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/htdocs/bloodhound.css
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_enums.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_logging.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_milestones.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_versions.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_ticket.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_ticket_box.html
incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/setup.py
Propchange: incubator/bloodhound/branches/bep_0003_multiproduct/
------------------------------------------------------------------------------
Merged /incubator/bloodhound/trunk:r1430288-1433320
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/NOTICE
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/NOTICE?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/NOTICE (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/NOTICE Tue Jan 15 09:32:53 2013
@@ -1,5 +1,5 @@
Apache Bloodhound
-Copyright 2012 The Apache Software Foundation
+Copyright 2012-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/)
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/RELEASE_NOTES
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/RELEASE_NOTES?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/RELEASE_NOTES (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/RELEASE_NOTES Tue Jan 15 09:32:53 2013
@@ -1,3 +1,14 @@
+0.4
+
+ * Replaces ticket edit form with a new 'in-place' edit and workflow control
+ * Added white-labelling for error messages and basic branding
+ * Improvements to the quick ticket creation form including ability to specify the select fields and their order.
+ * Various bug fixes
+
+ * Not fixed for this release:
+ * No major outstanding issues
+
+
0.3
* Removes minified files from Bloodhound
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_dashboard/setup.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_dashboard/setup.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_dashboard/setup.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_dashboard/setup.py Tue Jan 15 09:32:53 2013
@@ -32,6 +32,7 @@ versions = [
(0, 1, 0),
(0, 2, 0),
(0, 3, 0),
+ (0, 4, 0),
]
latest = '.'.join(str(x) for x in versions[-1])
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/setup.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/setup.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/setup.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/setup.py Tue Jan 15 09:32:53 2013
@@ -21,7 +21,7 @@ from setuptools import setup
setup(
name = 'BloodhoundMultiProduct',
- version = '0.3.0',
+ version = '0.4.0',
packages = ['multiproduct', 'multiproduct.ticket', 'tests',],
package_data = {'multiproduct' : ['templates/*.html',]},
entry_points = {'trac.plugins': [
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/api.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/api.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/api.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/api.py Tue Jan 15 09:32:53 2013
@@ -27,6 +27,13 @@ ASC = "asc"
DESC = "desc"
SCORE = "score"
+class IndexFields(object):
+ TYPE = "type"
+ ID = "id"
+ TIME = 'time'
+ AUTHOR = 'author'
+ CONTENT = 'content'
+
class QueryResult(object):
def __init__(self):
self.hits = 0
@@ -35,48 +42,50 @@ class QueryResult(object):
self.offset = 0
self.docs = []
self.facets = None
+ self.debug = {}
class ISearchBackend(Interface):
"""Extension point interface for search backend systems.
"""
- def add_doc(self, doc, commit=True):
+# def add_doc(self, doc, **kwargs):
+ def add_doc(doc, **kwargs):
"""
Called when new document instance must be added
-
- :param doc: document to add
- :param commit: flag if commit should be automatically called
"""
- def delete_doc(self, doc, commit=True):
+ def delete_doc(type, id, **kwargs):
"""
Delete document from index
+ """
- :param doc: document to delete
- :param commit: flag if commit should be automatically called
+ def optimize():
+ """
+ Optimize index if needed
"""
- def commit(self):
+ def commit(optimize, **kwargs):
"""
- Commits changes
+ Commit changes
"""
- def optimize(self):
+ def cancel(**kwargs):
"""
- Optimize index if needed
+ Cancel changes if possible
"""
- def recreate_index(self):
+ def recreate_index():
"""
Create a new index, if index exists, it will be deleted
"""
- def open_or_create_index_if_missing(self):
+ def open_or_create_index_if_missing():
"""
Open existing index, if index does not exist, create new one
"""
- def query(self, query, sort = None, fields = None, boost = None, filters = None,
+
+ def query(query, sort = None, fields = None, boost = None, filter = None,
facets = None, pagenum = 1, pagelen = 20):
"""
Perform query implementation
@@ -85,38 +94,42 @@ class ISearchBackend(Interface):
:param sort:
:param fields:
:param boost:
- :param filters:
+ :param filter:
:param facets:
:param pagenum:
:param pagelen:
- :return: TBD!!!
+ :return: ResultsPage
"""
- pass
+
+ def start_operation(self):
+ """Used to get arguments for batch operation withing single commit"""
+
+class IIndexParticipant(Interface):
+ """Extension point interface for components that should be searched.
+ """
+
+ def get_entries_for_index():
+ """List entities for index creation"""
class ISearchParticipant(Interface):
"""Extension point interface for components that should be searched.
"""
+ def format_search_results(contents):
+ """Called to see if the module wants to format the search results."""
def get_search_filters(req):
"""Called when we want to build the list of components with search.
Passes the request object to do permission checking."""
- pass
- def build_search_index(backend):
- """Called when we want to rebuild the entire index.
- :type backend: ISearchBackend
- """
- pass
-
- def format_search_results(contents):
- """Called to see if the module wants to format the search results."""
+ def get_title():
+ """Return resource title"""
class IQueryParser(Interface):
"""Extension point for Bloodhound Search query parser.
"""
def parse(query_string, req = None):
- pass
+ """Parse query from string"""
class BloodhoundSearchApi(Component):
"""Implements core indexing functionality, provides methods for
@@ -132,10 +145,11 @@ class BloodhoundSearchApi(Component):
'Name of the component implementing Bloodhound Search query \
parser.')
- search_participants = ExtensionPoint(ISearchParticipant)
+ index_participants = ExtensionPoint(IIndexParticipant)
- def query(self, query, req = None, sort = None, fields = None, boost = None, filters = None,
- facets = None, pagenum = 1, pagelen = 20):
+ def query(self, query, sort = None, fields = None,
+ boost = None, filter = None,
+ facets = None, pagenum = 1, pagelen = 20):
"""Return query result from an underlying search backend.
Arguments:
@@ -144,9 +158,9 @@ class BloodhoundSearchApi(Component):
:param sort: optional sorting
:param boost: optional list of fields with boost values e.g.
{âidâ: 1000, âsubjectâ :100, âdescriptionâ:10}.
- :param filters: optional list of terms. Usually can be cached by underlying
- search framework. For example {âtypeâ: âwikiâ}
- :param facets: optional list of facet terms, can be field or expression.
+ :param filter: optional list of terms. Usually can be cached by
+ underlying search framework. For example {âtypeâ: âwikiâ}
+ :param facets: optional list of facet terms, can be field or expression
:param page: paging support
:param pagelen: paging support
@@ -154,18 +168,17 @@ class BloodhoundSearchApi(Component):
"""
self.env.log.debug("Receive query request: %s", locals())
+ parsed_query = self.parser.parse(query)
+
# TODO: add query parsers and meta keywords post-parsing
# TODO: apply security filters
- parsed_query = self.parser.parse(query, req)
-
- #some backend-independent logic will come here...
query_result = self.backend.query(
query = parsed_query,
sort = sort,
fields = fields,
- filters = filters,
+ filter = filter,
facets = facets,
pagenum = pagenum,
pagelen = pagelen,
@@ -175,39 +188,64 @@ class BloodhoundSearchApi(Component):
def rebuild_index(self):
- """Delete the index if it exists. Then create a new full index."""
+ """Rebuild underlying index"""
self.log.info('Rebuilding the search index.')
self.backend.recreate_index()
+ operation_data = self.backend.start_operation()
+ try:
+ for participant in self.index_participants:
+ docs = participant.get_entries_for_index()
+ for doc in docs:
+ self.backend.add_doc(doc, **operation_data)
+ self.backend.commit(True, **operation_data)
+ except:
+ self.backend.cancel(**operation_data)
+ raise
+
+ def change_doc_id(self, doc, old_id):
+ operation_data = self.backend.start_operation()
+ try:
+ self.backend.delete_doc(
+ doc[IndexFields.TYPE],
+ old_id,
+ **operation_data)
+ self.backend.add_doc(doc, **operation_data)
+ self.backend.commit(False, **operation_data)
+ except:
+ self.backend.cancel(**operation_data)
+ raise
- for participant in self.search_participants:
- participant.build_search_index(self.backend)
- self.backend.commit()
- self.backend.optimize()
-
- #Erase the index if it exists. Then create a new index from scratch.
-
- #erase ticket
- #call reindex for each resource
- #commit
- pass
def optimize(self):
"""Optimize underlying index"""
- pass
+ self.backend.optimize()
def add_doc(self, doc):
"""Add a document to underlying search backend.
The doc must be dictionary with obligatory "type" field
"""
- self.backend.add_doc(doc)
+ operation_data = self.backend.start_operation()
+ try:
+ self.backend.add_doc(doc, **operation_data)
+ self.backend.commit(False, **operation_data)
+ except:
+ self.backend.cancel(**operation_data)
+ raise
+
def delete_doc(self, type, id):
"""Add a document from underlying search backend.
The doc must be dictionary with obligatory "type" field
"""
- pass
+ operation_data = self.backend.start_operation()
+ try:
+ self.backend.delete_doc(type, id, **operation_data)
+ self.backend.commit(False, **operation_data)
+ except:
+ self.backend.cancel(**operation_data)
+ raise
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/query_parser.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/query_parser.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/query_parser.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/query_parser.py Tue Jan 15 09:32:53 2013
@@ -23,12 +23,12 @@ r"""Provides Bloodhound Search query par
from bhsearch.api import IQueryParser
from bhsearch.whoosh_backend import WhooshBackend
from trac.core import Component, implements
-from whoosh.qparser import QueryParser, DisMaxParser, MultifieldParser
+from whoosh.qparser import MultifieldParser
class DefaultQueryParser(Component):
implements(IQueryParser)
- def parse(self, query_string, req = None):
+ def parse(self, query_string):
#todo: make field boost configurable e.g. read from config setting
#this is prototype implementation ,the fields boost must be tuned later
field_boosts = dict(
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/templates/bhsearch.html
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/templates/bhsearch.html?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/templates/bhsearch.html (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/templates/bhsearch.html Tue Jan 15 09:32:53 2013
@@ -43,16 +43,25 @@
<body>
<div id="content" class="search">
- <h1>This page provides prototype functionality. Implementation is coming...</h1>
+ <h1>This page provides prototype functionality.</h1>
<h1><label for="q">Search</label></h1>
<form id="fullsearch" action="${href.bhsearch()}" method="get">
<p>
<input type="text" id="q" name="q" size="40" value="${query}" />
<input type="hidden" name="noquickjump" value="1" />
+ <input py:if="active_type" type="hidden" name="type" value="${active_type}" />
<input type="submit" value="${_('Search')}" />
</p>
</form>
+ <div>
+ <ul class="nav nav-tabs" id="mainnav">
+ <li py:for="idx, item in enumerate(i for i in types)"
+ class="${classes(first_last(idx, types), active=item.active)}"><a href="${item.href}">${item.label}</a></li>
+
+ </ul>
+ </div>
+
<py:if test="results"><hr />
<h2 py:if="results">
Results <small>(${results.displayed_items()})</small>
@@ -60,18 +69,7 @@
<div>
<dl id="results">
- <!--This just a prototype stub. Should be replaced by proper ui mocks-->
- <div>
- <ul class="nav nav-tabs" id="mainnav">
- <!--<li py:if="chrome.nav.mainnav"-->
- <!--py:for="idx, item in enumerate(i for i in chrome.nav.mainnav if i.name in mainnav_show)" -->
- <!--class="${classes(first_last(idx, chrome.nav.mainnav), active=item.active)}">${item.label}</li>-->
- <li class="$active}"><a href="${page_href}">All (XXX)</a></li>
- <li class=""><a href="${page_href}">Wiki (XXX)</a></li>
- <li class=""><a href="${page_href}">Tickets (XXX)</a></li>
- </ul>
- </div>
-
+ <!--This just a prototype implementation. Should be replaced by proper UI mocks-->
<py:for each="result in results">
<dt><a href="${result.href}" class="searchable">${result.title}</a></dt>
<dd class="searchable">${result.excerpt}</dd>
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/__init__.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/__init__.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/__init__.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/__init__.py Tue Jan 15 09:32:53 2013
@@ -17,17 +17,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import doctest
import unittest
-from bhsearch.tests import whoosh_backend, index_with_whoosh, web_ui, ticket_search, api
+from bhsearch.tests import whoosh_backend, index_with_whoosh, web_ui, ticket_search, api, wiki_search
def suite():
suite = unittest.TestSuite()
suite.addTest(index_with_whoosh.suite())
suite.addTest(whoosh_backend.suite())
suite.addTest(web_ui.suite())
- suite.addTest(ticket_search.suite())
suite.addTest(api.suite())
+ suite.addTest(ticket_search.suite())
+ suite.addTest(wiki_search.suite())
return suite
if __name__ == '__main__':
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/api.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/api.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/api.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/api.py Tue Jan 15 09:32:53 2013
@@ -17,9 +17,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from datetime import datetime, timedelta
-from pprint import pprint
-
import unittest
import tempfile
import shutil
@@ -29,7 +26,7 @@ from bhsearch.tests.utils import BaseBlo
from bhsearch.ticket_search import TicketSearchParticipant
from bhsearch.whoosh_backend import WhooshBackend
-from trac.test import EnvironmentStub, Mock, MockPerm
+from trac.test import EnvironmentStub
from trac.ticket.api import TicketSystem
@@ -37,12 +34,12 @@ class ApiQueryWithWhooshTestCase(BaseBlo
def setUp(self):
self.env = EnvironmentStub(enable=['bhsearch.*'])
self.env.path = tempfile.mkdtemp('bhsearch-tempenv')
+ self.env.config.set('bhsearch', 'silence_on_error', "False")
self.ticket_system = TicketSystem(self.env)
self.whoosh_backend = WhooshBackend(self.env)
self.whoosh_backend.recreate_index()
self.search_api = BloodhoundSearchApi(self.env)
self.ticket_participant = TicketSearchParticipant(self.env)
- self.ticket_system = TicketSystem(self.env)
self.query_parser = DefaultQueryParser(self.env)
def tearDown(self):
@@ -103,17 +100,19 @@ class ApiQueryWithWhooshTestCase(BaseBlo
docs = results.docs
self.assertEqual("summary1 keyword", docs[0]["summary"])
- def test_can_search_id_and_summary(self):
- #arrange
- self.insert_ticket("summary1")
- self.insert_ticket("summary2 1")
- #act
- results = self.search_api.query("1")
- self.print_result(results)
- #assert
- self.assertEqual(2, results.hits)
- docs = results.docs
- self.assertEqual("summary1", docs[0]["summary"])
+#TODO: check this later
+# @unittest.skip("Check with Whoosh community")
+# def test_can_search_id_and_summary(self):
+# #arrange
+# self.insert_ticket("summary1")
+# self.insert_ticket("summary2 1")
+# #act
+# results = self.search_api.query("1")
+# self.print_result(results)
+# #assert
+# self.assertEqual(2, results.hits)
+# docs = results.docs
+# self.assertEqual("summary1", docs[0]["summary"])
def suite():
return unittest.makeSuite(ApiQueryWithWhooshTestCase, 'test')
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/index_with_whoosh.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/index_with_whoosh.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/index_with_whoosh.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/index_with_whoosh.py Tue Jan 15 09:32:53 2013
@@ -24,9 +24,10 @@ import tempfile
import shutil
from bhsearch.api import BloodhoundSearchApi
from bhsearch.tests.utils import BaseBloodhoundSearchTest
-from bhsearch.ticket_search import TicketSearchParticipant
+from bhsearch.ticket_search import TicketIndexer
from bhsearch.whoosh_backend import WhooshBackend
+from bhsearch.wiki_search import WikiIndexer
from trac.test import EnvironmentStub
from trac.ticket.api import TicketSystem
@@ -35,11 +36,11 @@ class IndexWhooshTestCase(BaseBloodhound
def setUp(self):
self.env = EnvironmentStub(enable=['bhsearch.*'])
self.env.path = tempfile.mkdtemp('bhsearch-tempenv')
-# self.perm = PermissionSystem(self.env)
self.whoosh_backend = WhooshBackend(self.env)
self.whoosh_backend.recreate_index()
self.search_api = BloodhoundSearchApi(self.env)
- self.ticket_participant = TicketSearchParticipant(self.env)
+ self.ticket_indexer = TicketIndexer(self.env)
+ self.wiki_indexer = WikiIndexer(self.env)
self.ticket_system = TicketSystem(self.env)
def tearDown(self):
@@ -49,7 +50,7 @@ class IndexWhooshTestCase(BaseBloodhound
def test_can_index_ticket(self):
ticket = self.create_dummy_ticket()
ticket.id = "1"
- self.ticket_participant.ticket_created(ticket)
+ self.ticket_indexer.ticket_created(ticket)
results = self.search_api.query("*:*")
self.print_result(results)
@@ -64,7 +65,18 @@ class IndexWhooshTestCase(BaseBloodhound
self.print_result(results)
self.assertEqual(1, results.hits)
- def test_can_reindex(self):
+ def test_can_reindex_twice(self):
+ self.insert_ticket("t1")
+ self.whoosh_backend.recreate_index()
+ #act
+ self.search_api.rebuild_index()
+ #just to test that index was re-created
+ self.search_api.rebuild_index()
+ #assert
+ results = self.search_api.query("*:*")
+ self.assertEqual(1, results.hits)
+
+ def test_can_reindex_tickets(self):
self.insert_ticket("t1")
self.insert_ticket("t2")
self.insert_ticket("t3")
@@ -76,6 +88,28 @@ class IndexWhooshTestCase(BaseBloodhound
self.print_result(results)
self.assertEqual(3, results.hits)
+ def test_can_reindex_wiki(self):
+ self.insert_wiki("page1", "some text")
+ self.insert_wiki("page2", "some text")
+ self.whoosh_backend.recreate_index()
+ #act
+ self.search_api.rebuild_index()
+ #assert
+ results = self.search_api.query("*:*")
+ self.print_result(results)
+ self.assertEqual(2, results.hits)
+
+ def test_can_reindex_mixed_types(self):
+ self.insert_wiki("page1", "some text")
+ self.insert_ticket("t1")
+ self.whoosh_backend.recreate_index()
+ #act
+ self.search_api.rebuild_index()
+ #assert
+ results = self.search_api.query("*:*")
+ self.print_result(results)
+ self.assertEqual(2, results.hits)
+
def suite():
return unittest.makeSuite(IndexWhooshTestCase, 'test')
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/ticket_search.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/ticket_search.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/ticket_search.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/ticket_search.py Tue Jan 15 09:32:53 2013
@@ -21,40 +21,36 @@ from datetime import datetime
import unittest
import tempfile
-import shutil
-from bhsearch.api import BloodhoundSearchApi
from bhsearch.tests.utils import BaseBloodhoundSearchTest
-from bhsearch.ticket_search import TicketSearchParticipant
+from bhsearch.ticket_search import TicketIndexer
-from bhsearch.whoosh_backend import WhooshBackend
from trac.test import EnvironmentStub
-from trac.ticket.api import TicketSystem
-class TicketSearchSilenceOnExceptionTestCase(BaseBloodhoundSearchTest):
+class TicketIndexerSilenceOnExceptionTestCase(BaseBloodhoundSearchTest):
def setUp(self):
self.env = EnvironmentStub(
enable=['bhsearch.*'],
path=tempfile.mkdtemp('bhsearch-tempenv'),
)
- self.ticket_participant = TicketSearchParticipant(self.env)
+ self.ticket_indexer = TicketIndexer(self.env)
def tearDown(self):
pass
def test_does_not_raise_exception_by_default(self):
- self.ticket_participant.ticket_created(None)
+ self.ticket_indexer.ticket_created(None)
def test_raise_exception_if_configured(self):
self.env.config.set('bhsearch', 'silence_on_error', "False")
self.assertRaises(
Exception,
- self.ticket_participant.ticket_created,
+ self.ticket_indexer.ticket_created,
None)
def suite():
- return unittest.makeSuite(TicketSearchSilenceOnExceptionTestCase, 'test')
+ return unittest.makeSuite(TicketIndexerSilenceOnExceptionTestCase, 'test')
if __name__ == '__main__':
unittest.main()
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/utils.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/utils.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/utils.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/utils.py Tue Jan 15 09:32:53 2013
@@ -24,6 +24,7 @@ Test utils methods
import pprint
import unittest
from trac.ticket import Ticket
+from trac.wiki import WikiPage
class BaseBloodhoundSearchTest(unittest.TestCase):
def print_result(self, result):
@@ -48,3 +49,16 @@ class BaseBloodhoundSearchTest(unittest.
ticket = self.create_ticket(summary, **kw)
return ticket.insert()
+ def create_wiki(self, name, text, **kw):
+ page = WikiPage(self.env, name)
+ page.text = text
+ for k, v in kw.items():
+ page[k] = v
+ return page
+
+ def insert_wiki(self, name, text = None, **kw):
+ """Helper for inserting a ticket into the database"""
+ text = text or "Dummy text"
+ page = self.create_wiki(name, text, **kw)
+ return page.save("dummy author", "dummy comment", "::1")
+
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/web_ui.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/web_ui.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/web_ui.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/web_ui.py Tue Jan 15 09:32:53 2013
@@ -17,7 +17,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from datetime import datetime, timedelta
from pprint import pprint
import unittest
@@ -26,7 +25,7 @@ import shutil
from bhsearch.api import BloodhoundSearchApi
from bhsearch.tests.utils import BaseBloodhoundSearchTest
from bhsearch.ticket_search import TicketSearchParticipant
-from bhsearch.web_ui import BloodhoundSearchModule, SEARCH_PERMISSION
+from bhsearch.web_ui import BloodhoundSearchModule, RequestParameters
from bhsearch.whoosh_backend import WhooshBackend
from trac.test import EnvironmentStub, Mock, MockPerm
@@ -36,7 +35,7 @@ from trac.util.datefmt import FixedOffse
from trac.util import format_datetime
from trac.web import Href
-BHSEARCH_URL = "/bhsearch"
+BHSEARCH_URL = "/main/bhsearch"
DEFAULT_DOCS_PER_PAGE = 10
class WebUiTestCaseWithWhoosh(BaseBloodhoundSearchTest):
@@ -56,19 +55,9 @@ class WebUiTestCaseWithWhoosh(BaseBloodh
self.req = Mock(
perm=MockPerm(),
chrome={'logo': {}},
- href=Href("/bhsearch"),
+ href=Href("/main"),
args={},
)
-# self.req = Mock(href=self.env.href, authname='anonymous', tz=utc)
-# self.req = Mock(base_path='/trac.cgi', path_info='',
-# href=Href('/trac.cgi'), chrome={'logo': {}},
-# abs_href=Href('http://example.org/trac.cgi'),
-# environ={}, perm=[], authname='-', args={}, tz=None,
-# locale='', session=None, form_token=None)
-
-# self.req = Mock(href=self.env.href, abs_href=self.env.abs_href, tz=utc,
-# perm=MockPerm())
-#
def tearDown(self):
shutil.rmtree(self.env.path)
@@ -89,14 +78,14 @@ class WebUiTestCaseWithWhoosh(BaseBloodh
self.assertEqual("", data["query"])
def test_can_process_query_empty_data(self):
- self.req.args["q"] = "bla"
+ self.req.args[RequestParameters.QUERY] = "bla"
data = self._process_request()
self.assertEqual("bla", data["query"])
self.assertEqual([], data["results"].items)
def test_can_process_first_page(self):
- self._insert_docs(5)
- self.req.args["q"] = "summary:test"
+ self._insert_tickets(5)
+ self.req.args[RequestParameters.QUERY] = "summary:test"
data = self._process_request()
self.assertEqual("summary:test", data["query"])
self.assertEqual(5, len(data["results"].items))
@@ -107,7 +96,7 @@ class WebUiTestCaseWithWhoosh(BaseBloodh
ticket = Ticket(self.env, ticket_id)
ticket_time = ticket.time_changed
#act
- self.req.args["q"] = "*:*"
+ self.req.args[RequestParameters.QUERY] = "*:*"
data = self._process_request()
result_items = data["results"].items
#assert
@@ -123,7 +112,7 @@ class WebUiTestCaseWithWhoosh(BaseBloodh
ticket_time = ticket.time_changed
#act
self.req.tz = FixedOffset(60, 'GMT +1:00')
- self.req.args["q"] = "*:*"
+ self.req.args[RequestParameters.QUERY] = "*:*"
data = self._process_request()
result_items = data["results"].items
#asset
@@ -135,31 +124,132 @@ class WebUiTestCaseWithWhoosh(BaseBloodh
self.assertEqual(expected_datetime, result_datetime)
def test_ticket_href(self):
- self._insert_docs(1)
- self.req.args["q"] = "*:*"
+ self._insert_tickets(1)
+ self.req.args[RequestParameters.QUERY] = "*:*"
data = self._process_request()
docs = data["results"].items
self.assertEqual(1, len(docs))
- self.assertEqual(BHSEARCH_URL + "/ticket/1", docs[0]["href"])
+ self.assertEqual("/main/ticket/1", docs[0]["href"])
def test_page_href(self):
- self._insert_docs(DEFAULT_DOCS_PER_PAGE+1)
- self.req.args["q"] = "*:*"
+ self._insert_tickets(DEFAULT_DOCS_PER_PAGE+1)
+ self.req.args[RequestParameters.QUERY] = "*:*"
data = self._process_request()
shown_pages = data["results"].shown_pages
- self.assertEqual(BHSEARCH_URL + "/bhsearch?q=*%3A*&page=2&noquickjump=1", shown_pages[1]["href"])
+ second_page_href = shown_pages[1]["href"]
+ self.assertIn("page=2", second_page_href)
+ self.assertIn("q=*%3A*", second_page_href)
- def test_facets(self):
+ def test_facets_ticket_only(self):
self.insert_ticket("summary1 keyword", status="closed")
self.insert_ticket("summary2 keyword", status="new")
- self.req.args["q"] = "*:*"
+ self.req.args[RequestParameters.QUERY] = "*:*"
data = self._process_request()
facets = data["facets"]
pprint(facets)
- self.assertEqual({u'ticket': 2}, facets["type"])
+ self.assertEqual({'ticket': 2}, facets["type"])
+ def test_facets_ticket_and_wiki(self):
+ self.insert_ticket("summary1 keyword", status="closed")
+ self.insert_ticket("summary2 keyword", status="new")
+ self.insert_wiki("dummyTitle", "Some text")
+ self.req.args[RequestParameters.QUERY] = "*"
+ data = self._process_request()
+ facets = data["facets"]
+ pprint(facets)
+ self.assertEqual({'ticket': 2, 'wiki': 1}, facets["type"])
+
+ def test_can_apply_type_parameter(self):
+ #arrange
+ self.insert_ticket("summary1 keyword", status="closed")
+ self.insert_ticket("summary2 keyword", status="new")
+ self.insert_wiki("dummyTitle", "Some text")
+ self.req.args[RequestParameters.QUERY] = "*"
+ self.req.args[RequestParameters.TYPE] = "ticket"
+ #act
+ data = self._process_request()
+ docs = data["results"].items
+ #assert
+ active_type = data["active_type"]
+ self.assertEquals("ticket", active_type)
+
+ resource_types = data["types"]
+
+ all = resource_types[0]
+ self._assertResourceType(all, "All", False)
+ self.assertNotIn("type", all["href"])
+
+ ticket = resource_types[1]
+ self._assertResourceType(ticket, "Ticket", True, "type=ticket")
+
+ wiki = resource_types[2]
+ self._assertResourceType(wiki, "Wiki", False, "type=wiki")
+
+ def test_type_parameter_in_links(self):
+ self._insert_tickets(12)
+ self.req.args[RequestParameters.QUERY] = "*"
+ self.req.args[RequestParameters.TYPE] = "ticket"
+ self.req.args[RequestParameters.PAGELEN] = "4"
+ self.req.args[RequestParameters.PAGE] = "2"
+ data = self._process_request()
+ results = data["results"]
+ docs = results.items
+ self.assertEquals(4, len(docs))
+
+ next_chrome_link = self.req.chrome['links']['next'][0]["href"]
+ self.assertIn('type=ticket', next_chrome_link)
+ self.assertIn('page=3', next_chrome_link)
+
+ prev_chrome_link = self.req.chrome['links']['prev'][0]["href"]
+ self.assertIn('type=ticket', prev_chrome_link)
+ self.assertIn('page=1', prev_chrome_link)
+
+ self.assertIn('type=ticket', data["page_href"])
+
+ for page in results.shown_pages:
+ self.assertIn('type=ticket', page["href"])
+
+ def test_type_grouping(self):
+ self.req.args[RequestParameters.QUERY] = "*:*"
+ data = self._process_request()
+ resource_types = data["types"]
+
+ all = resource_types[0]
+ self._assertResourceType(all, "All", True)
+ self.assertNotIn("type", all["href"])
+
+ ticket = resource_types[1]
+ self._assertResourceType(ticket, "Ticket", False, "type=ticket")
+
+ wiki = resource_types[2]
+ self._assertResourceType(wiki, "Wiki", False, "type=wiki")
+
+
+ def test_that_there_are_no_page_parameters_for_other_types(self):
+ #arrange
+ self._insert_tickets(12)
+ #act
+ self.req.args[RequestParameters.QUERY] = "*"
+ self.req.args[RequestParameters.PAGELEN] = "4"
+ self.req.args[RequestParameters.PAGE] = "2"
+ data = self._process_request()
+ #assert
+ resource_types = data["types"]
+
+ all = resource_types[0]
+ self.assertIn("page=2", all["href"])
+
+ ticket = resource_types[1]
+ self.assertNotIn("page=", ticket["href"])
+
+
+ def _assertResourceType(self, type, label, active, href_contains = None):
+ self.assertEquals(label, type["label"])
+ self.assertEquals(active, type["active"])
+ if href_contains:
+ self.assertIn(href_contains, type["href"])
- def _insert_docs(self, n):
+ def _insert_tickets(self, n):
for i in range(1, n+1):
self.insert_ticket("test %s" % i)
def suite():
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/whoosh_backend.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/whoosh_backend.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/whoosh_backend.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/tests/whoosh_backend.py Tue Jan 15 09:32:53 2013
@@ -23,20 +23,23 @@ import unittest
import tempfile
import shutil
from bhsearch.api import ASC, DESC, SCORE
+from bhsearch.query_parser import DefaultQueryParser
from bhsearch.tests.utils import BaseBloodhoundSearchTest
from bhsearch.whoosh_backend import WhooshBackend
from trac.test import EnvironmentStub
from trac.util.datefmt import FixedOffset, utc
-from whoosh.qparser import MultifieldParser, MultifieldPlugin, syntax, QueryParser, WhitespacePlugin, PhrasePlugin, PlusMinusPlugin
+from whoosh import index, sorting, query
+from whoosh.fields import Schema, ID, TEXT, KEYWORD
+from whoosh.qparser import MultifieldPlugin, QueryParser, WhitespacePlugin, PhrasePlugin
class WhooshBackendTestCase(BaseBloodhoundSearchTest):
def setUp(self):
self.env = EnvironmentStub(enable=['bhsearch.*'])
self.env.path = tempfile.mkdtemp('bhsearch-tempenv')
-# self.perm = PermissionSystem(self.env)
self.whoosh_backend = WhooshBackend(self.env)
self.whoosh_backend.recreate_index()
+ self.default_parser = DefaultQueryParser(self.env)
def tearDown(self):
shutil.rmtree(self.env.path)
@@ -46,8 +49,7 @@ class WhooshBackendTestCase(BaseBloodhou
self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
self.whoosh_backend.add_doc(dict(id="2", type="ticket"))
result = self.whoosh_backend.query(
-# result = self.search_api.query(
- "*:*",
+ query.Every(),
sort = [("id", ASC)],
)
self.print_result(result)
@@ -64,7 +66,7 @@ class WhooshBackendTestCase(BaseBloodhou
def test_can_return_all_fields(self):
self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
- result = self.whoosh_backend.query("*:*")
+ result = self.whoosh_backend.query(query.Every())
self.print_result(result)
docs = result.docs
self.assertEqual(
@@ -74,7 +76,7 @@ class WhooshBackendTestCase(BaseBloodhou
def test_can_select_fields(self):
self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
- result = self.whoosh_backend.query("*:*",
+ result = self.whoosh_backend.query(query.Every(),
fields=("id", "type"))
self.print_result(result)
docs = result.docs
@@ -87,7 +89,7 @@ class WhooshBackendTestCase(BaseBloodhou
self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
whoosh_backend2 = WhooshBackend(self.env)
whoosh_backend2.add_doc(dict(id="2", type="ticket"))
- result = whoosh_backend2.query("*:*")
+ result = whoosh_backend2.query(query.Every())
self.assertEqual(2, result.hits)
def test_can_multi_sort_asc(self):
@@ -96,7 +98,7 @@ class WhooshBackendTestCase(BaseBloodhou
self.whoosh_backend.add_doc(dict(id="4", type="ticket3"))
self.whoosh_backend.add_doc(dict(id="1", type="ticket1"))
result = self.whoosh_backend.query(
- "*:*",
+ query.Every(),
sort = [("type", ASC), ("id", ASC)],
fields=("id", "type"),
)
@@ -113,7 +115,7 @@ class WhooshBackendTestCase(BaseBloodhou
self.whoosh_backend.add_doc(dict(id="4", type="ticket3"))
self.whoosh_backend.add_doc(dict(id="1", type="ticket1"))
result = self.whoosh_backend.query(
- "*:*",
+ query.Every(),
sort = [("type", ASC), ("id", DESC)],
fields=("id", "type"),
)
@@ -156,10 +158,12 @@ class WhooshBackendTestCase(BaseBloodhou
summary="some text out of search scope",
time=the_third_date,
))
+
+ parsed_query = self.default_parser.parse("summary:texttofind")
+
result = self.whoosh_backend.query(
- "summary:texttofind",
+ parsed_query,
sort = [(SCORE, ASC), ("time", DESC)],
-# fields=("id", "type"),
)
self.print_result(result)
self.assertEqual(3, result.hits)
@@ -176,7 +180,7 @@ class WhooshBackendTestCase(BaseBloodhou
self.whoosh_backend.add_doc(dict(id="2", type="ticket", product="B"))
self.whoosh_backend.add_doc(dict(id="3", type="wiki", product="A"))
result = self.whoosh_backend.query(
- "*:*",
+ query.Every(),
sort = [("type", ASC), ("id", DESC)],
fields=("id", "type"),
facets= ("type", "product")
@@ -187,25 +191,22 @@ class WhooshBackendTestCase(BaseBloodhou
self.assertEqual({"ticket":2, "wiki":1}, facets["type"])
self.assertEqual({"A":2, "B":1}, facets["product"])
- @unittest.skip(
- "Fix this, check why exception is raise on Whoosh mailing list")
- #TODO: fix this!!!!
def test_can_do_facet_if_filed_missing_TODO(self):
self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
self.whoosh_backend.add_doc(dict(id="2", type="ticket", status="New"))
result = self.whoosh_backend.query(
- "*:*",
+ query.Every(),
facets= ("type", "status")
)
self.print_result(result)
self.assertEqual(2, result.hits)
facets = result.facets
self.assertEqual({"ticket":2}, facets["type"])
- self.assertEqual({"new":1}, facets["status"])
+ self.assertEqual({None: 1, 'New': 1}, facets["status"])
def test_can_return_empty_result(self):
result = self.whoosh_backend.query(
- "*:*",
+ query.Every(),
sort = [("type", ASC), ("id", DESC)],
fields=("id", "type"),
facets= ("type", "product")
@@ -216,14 +217,14 @@ class WhooshBackendTestCase(BaseBloodhou
def test_can_search_time_with_utc_tzinfo(self):
time = datetime(2012, 12, 13, 11, 8, 34, 711957, tzinfo=FixedOffset(0, 'UTC'))
self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time))
- result = self.whoosh_backend.query("*:*")
+ result = self.whoosh_backend.query(query.Every())
self.print_result(result)
self.assertEqual(time, result.docs[0]["time"])
def test_can_search_time_without_tzinfo(self):
time = datetime(2012, 12, 13, 11, 8, 34, 711957, tzinfo=None)
self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time))
- result = self.whoosh_backend.query("*:*")
+ result = self.whoosh_backend.query(query.Every())
self.print_result(result)
self.assertEqual(time.replace(tzinfo=utc), result.docs[0]["time"])
@@ -233,60 +234,139 @@ class WhooshBackendTestCase(BaseBloodhou
time = datetime(2012, 12, 13, 11, hours, 34, 711957,
tzinfo=FixedOffset(tz_diff, "just_one_timezone"))
self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time))
- result = self.whoosh_backend.query("*:*")
+ result = self.whoosh_backend.query(query.Every())
self.print_result(result)
self.assertEqual(datetime(2012, 12, 13, 11, hours-tz_diff, 34, 711957,
tzinfo=utc), result.docs[0]["time"])
+
+ def test_can_apply_filter_and_facet(self):
+ self.whoosh_backend.add_doc(dict(id="1", type="ticket"))
+ self.whoosh_backend.add_doc(dict(id="2", type="wiki" ))
+ result = self.whoosh_backend.query(
+ query.Every(),
+ filter=[("type", "ticket")],
+ facets=["type"]
+ )
+ self.print_result(result)
+ self.assertEqual(1, result.hits)
+ self.assertEqual("ticket", result.docs[0]["type"])
+
+
@unittest.skip("TODO clarify behavior on Whoosh mail list")
- def test_can_search_id_and_summary(self):
+ def test_can_search_id_and_summary_TODO(self):
#arrange
self.insert_ticket("test x")
self.insert_ticket("test 1")
-# field_boosts = dict(
-# id = 6,
-# type = 2,
-# summary = 5,
-# author = 3,
-# milestone = 2,
-# keywords = 2,
-# component = 2,
-# status = 2,
-# content = 1,
-# changes = 0.8,
-# )
fieldboosts = dict(
id = 1,
summary = 1,
)
-# parser = MultifieldParser(
-# fieldboosts.keys(),
-# WhooshBackend.SCHEMA,
-## fieldboosts=field_boosts
-# )
-
- mfp = MultifieldPlugin(list(fieldboosts.keys()),
-# fieldboosts=fieldboosts,
-# group=syntax.DisMaxGroup
- )
+ mfp = MultifieldPlugin(list(fieldboosts.keys()),)
pins = [WhitespacePlugin,
-# PlusMinusPlugin,
PhrasePlugin,
mfp]
parser = QueryParser(None, WhooshBackend.SCHEMA, plugins=pins)
-
- parsed_query = parser.parse(u"1")
-# parsed_query = parser.parse(u"test")
+ parsed_query = parser.parse("1")
result = self.whoosh_backend.query(parsed_query)
self.print_result(result)
self.assertEqual(2, result.hits)
+class WhooshFunctionalityTestCase(unittest.TestCase):
+ def setUp(self):
+ self.index_dir = tempfile.mkdtemp('whoosh_index')
+
+ def tearDown(self):
+ shutil.rmtree(self.index_dir)
+
+ def test_groupedby_empty_field(self):
+ schema = Schema(
+ unique_id=ID(stored=True, unique=True),
+ id=ID(stored=True),
+ type=ID(stored=True),
+ status=KEYWORD(stored=True),
+ content=TEXT(stored=True),
+ )
+
+ ix = index.create_in(self.index_dir, schema=schema)
+ with ix.writer() as w:
+ w.add_document(unique_id="1",type="type1")
+ w.add_document(unique_id="2",type="type2", status="New")
+
+ facet_fields = ("type", "status" )
+ groupedby = facet_fields
+ with ix.searcher() as s:
+ r = s.search(
+ query.Every(),
+ groupedby=groupedby,
+ maptype=sorting.Count,
+ )
+ facets = self._load_facets(r)
+ print len(r) == 2
+ print facets
+ self.assertEquals(
+ {'status': {None: 1, 'New': 1}, 'type': {'type1': 1, 'type2': 1}},
+ facets)
+
+ def test_groupedby_empty_field(self):
+ """
+ Whoosh 2.4 raises an error when simultaneously using filters and facets
+ in search:
+ AttributeError: 'FacetCollector' object has no attribute 'offset'
+
+ The problem should be fixed in the next release. For more info read
+ https://bitbucket.org/mchaput/whoosh/issue/274
+
+ For the time of being, whoosh-backend have to introduce workaround in
+ order to fix the problem. This unit-test is just a reminder to remove
+ workaround when the fixed version of Whoosh is applied.
+ """
+ schema = Schema(
+ unique_id=ID(stored=True, unique=True),
+ type=ID(stored=True),
+ )
+
+ ix = index.create_in(self.index_dir, schema=schema)
+ with ix.writer() as w:
+ w.add_document(unique_id=u"1",type=u"type1")
+ w.add_document(unique_id=u"2",type=u"type2")
+
+ with ix.searcher() as s:
+ with self.assertRaises(AttributeError):
+ s.search(
+ query.Every(),
+ groupedby=("type"),
+ maptype=sorting.Count,
+ filter=query.Term("type", "type1")
+ )
+
+# def _prepare_groupedby(self, facets):
+# if not facets:
+# return None
+# groupedby = sorting.Facets()
+# for facet_name in facets:
+# groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count)
+# return groupedby
+
+ def _load_facets(self, non_paged_results):
+ facet_names = non_paged_results.facet_names()
+ if not facet_names:
+ return None
+ facets_result = dict()
+ for name in facet_names:
+ facets_result[name] = non_paged_results.groups(name)
+ return facets_result
+
+
def suite():
- return unittest.makeSuite(WhooshBackendTestCase, 'test')
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(WhooshBackendTestCase, 'test'))
+ suite.addTest(unittest.makeSuite(WhooshFunctionalityTestCase, 'test'))
+ return suite
if __name__ == '__main__':
unittest.main()
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/ticket_search.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/ticket_search.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/ticket_search.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/ticket_search.py Tue Jan 15 09:32:53 2013
@@ -19,60 +19,22 @@
# under the License.
r"""Ticket specifics for Bloodhound Search plugin."""
-from bhsearch.api import ISearchParticipant, BloodhoundSearchApi
+from bhsearch.api import ISearchParticipant, BloodhoundSearchApi, IIndexParticipant, IndexFields
from genshi.builder import tag
from trac.core import *
-from trac.ticket.api import ITicketChangeListener, TicketSystem
+from trac.ticket.api import ITicketChangeListener
from trac.ticket import Ticket
from trac.ticket.query import Query
from trac.config import Option
-from trac.web.chrome import add_warning
-from trac.util.datefmt import to_datetime
-TICKET = "ticket"
-TICKET_STATUS = 'status'
+TICKET_TYPE = "ticket"
+TICKET_STATUS = "status"
-class TicketSearchParticipant(Component):
- implements(ITicketChangeListener, ISearchParticipant)
+class TicketIndexer(Component):
+ implements(ITicketChangeListener, IIndexParticipant)
silence_on_error = Option('bhsearch', 'silence_on_error', "True",
"""If true, do not throw an exception during indexing a resource""")
- def _index_ticket(self, ticket, search_api=None, raise_exception = False):
- """Internal method for actually indexing a ticket.
- This reduces duplicating code."""
- try:
- if not search_api:
- search_api = BloodhoundSearchApi(self.env)
-
- #This is very naive prototype implementation
- #TODO: a lot of improvements must be added here!!!
- contents = {
- 'id': unicode(ticket.id),
- 'time': ticket.time_changed,
- 'type': TICKET,
- }
- fields = [('component',), ('description','content'), ('component',),
- ('keywords',), ('milestone',), ('summary',),
- ('status',), ('resolution',), ('reporter','author')]
- for f in fields:
- if f[0] in ticket.values:
- if len(f) == 1:
- contents[f[0]] = ticket.values[f[0]]
- elif len(f) == 2:
- contents[f[1]] = ticket.values[f[0]]
- contents['changes'] = u'\n\n'.join([x[4] for x in ticket.get_changelog()
- if x[2] == u'comment'])
- search_api.add_doc(contents)
- except Exception, e:
- if (not raise_exception) and self.silence_on_error.lower() == "true":
- #Is there any way to get request object to add warning?
- # add_warning(req, _('Exception during ticket indexing: %s' % e))
- self.log.error("Error occurs during ticke indexing. \
- The error will not be propagated. Exception: %s", e)
- else:
- raise
-
-
#ITicketChangeListener methods
def ticket_created(self, ticket):
"""Index a recently created ticket."""
@@ -84,31 +46,82 @@ class TicketSearchParticipant(Component)
def ticket_deleted(self, ticket):
"""Called when a ticket is deleted."""
- s = BloodhoundSearchApi(self.env)
- s.delete_doc(u'ticket', unicode(ticket.id))
+ try:
+ search_api = BloodhoundSearchApi(self.env)
+ search_api.delete_doc(TICKET_TYPE, ticket.id)
+ except Exception, e:
+ if self.silence_on_error.lower() == "true":
+ self.log.error("Error occurs during ticket indexing. \
+ The error will not be propagated. Exception: %s", e)
+ else:
+ raise
- # ISearchParticipant methods
- def get_search_filters(self, req=None):
- if not req or 'TICKET_VIEW' in req.perm:
- return ('ticket', 'Tickets')
+ def _index_ticket(
+ self,
+ ticket,
+ raise_exception = False,
+ ):
+ try:
+ search_api = BloodhoundSearchApi(self.env)
+ doc = self.build_doc(ticket)
+ search_api.add_doc(doc)
+ except Exception, e:
+ if (not raise_exception) and self.silence_on_error.lower() == "true":
+ self.log.error("Error occurs during ticket indexing. \
+ The error will not be propagated. Exception: %s", e)
+ else:
+ raise
- def build_search_index(self, backend):
- """
- :type backend: ISearchBackend
- """
- #TODO: some king of paging/batch size should be introduced in order to
- # avoid loading of all ticket ids in memory
- query_records = self.load_tickets_ids()
+ #IIndexParticipant members
+ def build_doc(self, trac_doc):
+ ticket = trac_doc
+ doc = {
+ IndexFields.ID: unicode(ticket.id),
+ IndexFields.TYPE: TICKET_TYPE,
+ IndexFields.TIME: ticket.time_changed,
+ }
+ fields = [('component',),
+ ('description',IndexFields.CONTENT),
+ ('keywords',),
+ ('milestone',),
+ ('summary',),
+ ('status',),
+ ('resolution',),
+ ('reporter',IndexFields.AUTHOR),
+ ]
+ for f in fields:
+ if f[0] in ticket.values:
+ if len(f) == 1:
+ doc[f[0]] = ticket.values[f[0]]
+ elif len(f) == 2:
+ doc[f[1]] = ticket.values[f[0]]
+ doc['changes'] = u'\n\n'.join([x[4] for x in ticket.get_changelog()
+ if x[2] == u'comment'])
+ return doc
+
+ def get_entries_for_index(self):
+ #is there any better way to get all tickets?
+ query_records = self._load_ticket_ids()
for record in query_records:
- ticket_id = record["id"]
- ticket = Ticket(self.env, ticket_id)
- self._index_ticket(ticket, backend, raise_exception=True)
+ ticket = Ticket(self.env, record["id"])
+ yield self.build_doc(ticket)
- def load_tickets_ids(self):
- #is there better way to get all tickets?
+ def _load_ticket_ids(self):
query = Query(self.env, cols=['id'], order='id')
return query.execute()
+
+class TicketSearchParticipant(Component):
+ implements(ISearchParticipant)
+
+ #ISearchParticipant members
+ def get_search_filters(self, req=None):
+ if not req or 'TICKET_VIEW' in req.perm:
+ return TICKET_TYPE
+
+ def get_title(self):
+ return "Ticket"
+
def format_search_results(self, res):
if not TICKET_STATUS in res:
stat = 'undefined_status'
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/web_ui.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/web_ui.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/web_ui.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/web_ui.py Tue Jan 15 09:32:53 2013
@@ -27,16 +27,85 @@ from trac.core import *
from genshi.builder import tag
from trac.perm import IPermissionRequestor
from trac.search import shorten_result
+from trac.config import OrderedExtensionsOption
from trac.util.presentation import Paginator
from trac.util.datefmt import format_datetime, user_time
from trac.web import IRequestHandler
from trac.util.translation import _
-from trac.web.chrome import (INavigationContributor, ITemplateProvider,
- add_link, add_stylesheet, add_warning,
- web_context)
-from bhsearch.api import BloodhoundSearchApi, ISearchParticipant, SCORE, ASC, DESC
+from trac.web.chrome import INavigationContributor, ITemplateProvider, \
+ add_link, add_stylesheet
+from bhsearch.api import BloodhoundSearchApi, ISearchParticipant, SCORE, ASC, \
+ DESC, IndexFields
SEARCH_PERMISSION = 'SEARCH_VIEW'
+DEFAULT_RESULTS_PER_PAGE = 10
+DEFAULT_SORT = [(SCORE, ASC), ("time", DESC)]
+
+
+class RequestParameters(object):
+ """
+ Helper class for parameter parsing and creation of bhsearch specific URLs.
+
+ Lifecycle of the class must be per request
+ """
+ QUERY = "q"
+ PAGE = "page"
+ FILTER = "fl"
+ TYPE = "type"
+ NO_QUICK_JUMP = "noquickjump"
+ PAGELEN = "pagelen"
+
+ def __init__(self, req):
+ self.req = req
+
+ self.query = req.args.get(RequestParameters.QUERY)
+ if self.query == None:
+ self.query = ""
+
+ #TODO: add quick jump functionality
+ self.noquickjump = 1
+
+ #TODO: add filters support
+ self.filters = []
+
+ #TODO: retrieve sort from query string
+ self.sort = DEFAULT_SORT
+
+ self.pagelen = int(req.args.get(
+ RequestParameters.PAGELEN,
+ DEFAULT_RESULTS_PER_PAGE))
+ self.page = int(req.args.get(RequestParameters.PAGE, '1'))
+ self.type = req.args.get(RequestParameters.TYPE, None)
+
+ self.params = {
+ self.NO_QUICK_JUMP: self.noquickjump,
+ }
+ if self.query:
+ self.params[self.QUERY] = self.query
+ if self.pagelen != DEFAULT_RESULTS_PER_PAGE:
+ self.params[self.PAGELEN]=self.pagelen
+ if self.page > 1:
+ self.params[self.PAGE]=self.page
+ if self.type:
+ self.params[self.TYPE] = self.type
+
+ def create_href(self, page = None, type=None, skip_type = False,
+ skip_page = False):
+ params = dict(self.params)
+ if page:
+ params[self.PAGE] = page
+
+ if skip_page and self.PAGE in params:
+ del(params[self.PAGE])
+
+ if type:
+ params[self.TYPE] = type
+
+ if skip_type and self.TYPE in params:
+ #show all does not require type parameter
+ del(params[self.TYPE])
+
+ return self.req.href.bhsearch(**params)
class BloodhoundSearchModule(Component):
"""Main search page"""
@@ -46,10 +115,12 @@ class BloodhoundSearchModule(Component):
# IWikiSyntaxProvider #todo: implement later
)
- search_participants = ExtensionPoint(ISearchParticipant)
-
- RESULTS_PER_PAGE = 10
- DEFAULT_SORT = [(SCORE, ASC), ("time", DESC)]
+ search_participants = OrderedExtensionsOption(
+ 'bhsearch',
+ 'search_participants',
+ ISearchParticipant,
+ "TicketSearchParticipant, WikiSearchParticipant"
+ )
# INavigationContributor methods
@@ -59,119 +130,150 @@ class BloodhoundSearchModule(Component):
def get_navigation_items(self, req):
if SEARCH_PERMISSION in req.perm:
yield ('mainnav', 'bhsearch',
- tag.a(_('Bloodhound Search'), href=self.env.href.bhsearch()))
+ tag.a(_('Bloodhound Search'), href=self.env.href.bhsearch())
+ )
# IPermissionRequestor methods
def get_permission_actions(self):
return [SEARCH_PERMISSION]
# IRequestHandler methods
-
def match_request(self, req):
return re.match(r'/bhsearch?', req.path_info) is not None
def process_request(self, req):
req.perm.assert_permission(SEARCH_PERMISSION)
-
- query = req.args.get('q')
- if query == None:
- query = ""
+ parameters = RequestParameters(req)
#TODO add quick jump support
- #TODO: refactor filters or replace with facets
- filters = []
-# available_filters = filter(None, [p.get_search_filters(req) for p
-# in self.search_participants])
-# filters = [f[0] for f in available_filters if req.args.has_key(f[0])]
-# if not filters:
-# filters = [f[0] for f in available_filters
-# if f[0] not in self.default_disabled_filters and
-# (len(f) < 3 or len(f) > 2 and f[2])]
-# data = {'filters': [{'name': f[0], 'label': f[1],
-# 'active': f[0] in filters}
-# for f in available_filters],
-# 'quickjump': None,
-# 'results': []}
-
+ allowed_participants = self._get_allowed_participants(req)
data = {
- 'query': query,
- }
+ 'query': parameters.query,
+ }
- # Initial page request
#todo: filters check, tickets etc
- if not any((query, )):
+ if not any((parameters.query, )):
return self._return_data(req, data)
- page = int(req.args.get('page', '1'))
-
- #todo: retrieve sort from query string
- sort = self.DEFAULT_SORT
+ query_filter = self._prepare_query_filter(
+ parameters.type,
+ parameters.filters,
+ allowed_participants)
#todo: add proper facets functionality
-# facets = ("type", "status")
- facets = ("type",)
-
+ facets = self._prepare_facets(req)
querySystem = BloodhoundSearchApi(self.env)
query_result = querySystem.query(
- query,
- pagenum = page,
- pagelen = self.RESULTS_PER_PAGE,
- sort = sort,
+ parameters.query,
+ pagenum = parameters.page,
+ pagelen = parameters.pagelen,
+ sort = parameters.sort,
facets = facets,
- )
- ui_docs = [self._process_doc(doc, req)
- for doc in query_result.docs]
+ filter=query_filter)
+ ui_docs = [self._process_doc(doc, req, allowed_participants)
+ for doc in query_result.docs]
results = Paginator(
ui_docs,
- page - 1,
- self.RESULTS_PER_PAGE,
- query_result.hits,
- )
+ parameters.page - 1,
+ parameters.pagelen,
+ query_result.hits)
results.shown_pages = self._prepare_shown_pages(
- filters,
- query,
- req,
- shown_pages = results.get_shown_pages(self.RESULTS_PER_PAGE))
+ parameters,
+ shown_pages = results.get_shown_pages(parameters.pagelen))
- results.current_page = {'href': None, 'class': 'current',
+ results.current_page = {'href': None,
+ 'class': 'current',
'string': str(results.page + 1),
'title':None}
if results.has_next_page:
- next_href = req.href.bhsearch(zip(filters, ['on'] * len(filters)),
- q=req.args.get('q'), page=page + 1,
- noquickjump=1)
+ next_href = parameters.create_href(page = parameters.page + 1)
add_link(req, 'next', next_href, _('Next Page'))
if results.has_previous_page:
- prev_href = req.href.bhsearch(zip(filters, ['on'] * len(filters)),
- q=req.args.get('q'), page=page - 1,
- noquickjump=1)
+ prev_href = parameters.create_href(page = parameters.page - 1)
add_link(req, 'prev', prev_href, _('Previous Page'))
data['results'] = results
+ self._prepare_type_grouping(
+ allowed_participants,
+ parameters,
+ data)
- #add proper facet links
+ #TODO:add proper facet links
data['facets'] = query_result.facets
-
- data['page_href'] = req.href.bhsearch(
- zip(filters, ['on'] * len(filters)), q=req.args.get('q'),
- noquickjump=1)
+ data['page_href'] = parameters.create_href()
return self._return_data(req, data)
+ def _prepare_query_filter(self, type, filters, allowed_participants):
+ query_filters = []
+
+ if type in allowed_participants:
+ query_filters.append((IndexFields.TYPE, type))
+ else:
+ self.log.debug("Unsupported type in web request: %s", type)
+
+ #TODO: handle other filters
+ return query_filters
+
+ def _prepare_type_grouping(self, allowed_participants, parameters, data):
+ active_type = parameters.type
+ if active_type and active_type not in allowed_participants:
+ raise TracError(_("Unsupported resource type: '%(name)s'",
+ name=active_type))
+ all_is_active = (active_type is None)
+ grouping = [
+ dict(
+ label=_("All"),
+ active=all_is_active,
+ href=parameters.create_href(
+ skip_type=True,
+ skip_page=not all_is_active)
+ )
+ ]
+
+ #we want to obtain the same order as specified in search_participants
+ # option
+ participant_with_type = dict((participant, type)
+ for type, participant in allowed_participants.iteritems())
+ for participant in self.search_participants:
+ if participant in participant_with_type:
+ type = participant_with_type[participant]
+ is_active = (type == active_type)
+ grouping.append(dict(
+ label=_(participant.get_title()),
+ active=is_active,
+ href=parameters.create_href(
+ type=type,
+ skip_page=not is_active
+ )
+ ))
+ data["types"] = grouping
+ data["active_type"] = active_type
+
+ def _prepare_facets(self, req):
+ facets = [IndexFields.TYPE]
+ #TODO: add type specific default facets
+ return facets
+
+ def _get_allowed_participants(self, req):
+ allowed_participants = {}
+ for participant in self.search_participants:
+ type = participant.get_search_filters(req)
+ if type is not None:
+ allowed_participants[type] = participant
+ return allowed_participants
+
def _return_data(self, req, data):
add_stylesheet(req, 'common/css/search.css')
return 'bhsearch.html', data, None
- def _process_doc(self, doc,req):
- titlers = dict([(x.get_search_filters(req)[0], x.format_search_results)
- for x in self.search_participants if x.get_search_filters(req)])
-
+ def _process_doc(self, doc, req, allowed_participants):
#todo: introduce copy by predefined value
ui_doc = dict(doc)
@@ -182,25 +284,25 @@ class BloodhoundSearchModule(Component):
if doc.has_key('time'):
ui_doc['date'] = user_time(req, format_datetime, doc['time'])
- ui_doc['title'] = titlers[doc['type']](doc)
+ ui_doc['title'] = allowed_participants[doc['type']]\
+ .format_search_results(doc)
return ui_doc
- def _prepare_shown_pages(self, filters, query, req, shown_pages):
- pagedata = []
+ def _prepare_shown_pages(self, parameters, shown_pages):
+ page_data = []
for shown_page in shown_pages:
- page_href = req.href.bhsearch([(f, 'on') for f in filters],
- q=query,
- page=shown_page, noquickjump=1)
- pagedata.append([page_href, None, str(shown_page),
+ page_href = parameters.create_href(page=shown_page)
+ page_data.append([page_href, None, str(shown_page),
'page ' + str(shown_page)])
fields = ['href', 'class', 'string', 'title']
- result_shown_pages = [dict(zip(fields, p)) for p in pagedata]
+ result_shown_pages = [dict(zip(fields, p)) for p in page_data]
return result_shown_pages
# ITemplateProvider methods
def get_htdocs_dirs(self):
-# return [('bhsearch', pkg_resources.resource_filename(__name__, 'htdocs'))]
+# return [('bhsearch',
+# pkg_resources.resource_filename(__name__, 'htdocs'))]
return []
def get_templates_dirs(self):
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/whoosh_backend.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/whoosh_backend.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/whoosh_backend.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/bhsearch/whoosh_backend.py Tue Jan 15 09:32:53 2013
@@ -22,15 +22,15 @@ r"""Whoosh specific backend for Bloodhou
from bhsearch.api import ISearchBackend, DESC, QueryResult, SCORE
import os
from trac.core import *
-from trac.config import Option, PathOption
+from trac.config import Option
from trac.util.datefmt import utc
from whoosh.fields import *
-from whoosh import index, sorting
-from whoosh.qparser import QueryParser
+from whoosh import index, sorting, query
from whoosh.searching import ResultsPage
-from whoosh.sorting import FieldFacet
from whoosh.writing import AsyncWriter
-from datetime import datetime, date
+from datetime import datetime
+
+UNIQUE_ID = 'unique_id'
class WhooshBackend(Component):
"""
@@ -68,7 +68,13 @@ class WhooshBackend(Component):
self.open_or_create_index_if_missing()
#ISearchBackend methods
- def add_doc(self, doc, commit=True):
+ def start_operation(self):
+ return dict(writer = self._create_writer())
+
+ def _create_writer(self):
+ return AsyncWriter(self.index)
+
+ def add_doc(self, doc, writer=None):
"""Add any type of document index.
The contents should be a dict with fields matching the search schema.
@@ -76,74 +82,126 @@ class WhooshBackend(Component):
"""
# Really make sure it's unicode, because Whoosh won't have it any
# other way.
+ is_local_writer = False
+ if writer is None:
+ is_local_writer = True
+ writer = self._create_writer()
+
for key in doc:
doc[key] = self._to_whoosh_format(doc[key])
+ doc["unique_id"] = self._create_unique_id(doc["type"], doc["id"])
+ self.log.debug("Doc to index: %s", doc)
+ try:
+ writer.update_document(**doc)
+ if is_local_writer:
+ writer.commit()
+ except:
+ if is_local_writer:
+ writer.cancel()
- doc["unique_id"] = u"%s:%s" % (doc["type"], doc["id"])
+ def delete_doc(self, type, id, writer=None):
+ unique_id = self._create_unique_id(type, id)
+ self.log.debug('Removing document from the index: %s', unique_id)
+ is_local_writer = False
+ if writer is None:
+ is_local_writer = True
+ writer = self._create_writer()
+ try:
+ writer.delete_by_term(UNIQUE_ID, unique_id)
+ if is_local_writer:
+ writer.commit()
+ except:
+ if is_local_writer:
+ writer.cancel()
+ raise
+
+ def optimize(self):
writer = AsyncWriter(self.index)
- committed = False
+ writer.commit(optimize=True)
+
+ def commit(self, optimize, writer):
+ writer.commit(optimize=optimize)
+
+ def cancel(self, writer):
try:
- #todo: remove it!!!
- self.log.debug("Doc to index: %s", doc)
- writer.update_document(**doc)
- writer.commit()
- committed = True
- finally:
- if not committed:
- writer.cancel()
+ writer.cancel()
+ except Exception,ex:
+ self.env.log.error("Error during writer cancellation: %s", ex)
+ def recreate_index(self):
+ self.log.info('Creating Whoosh index in %s' % self.index_dir)
+ self._make_dir_if_not_exists()
+ return index.create_in(self.index_dir, schema=self.SCHEMA)
- def query(self, query, sort = None, fields = None, boost = None, filters = None,
+ def open_or_create_index_if_missing(self):
+ if index.exists_in(self.index_dir):
+ self.index = index.open_dir(self.index_dir)
+ else:
+ self.index = self.recreate_index()
+
+ def query(self, query, sort = None, fields = None, boost = None, filter = None,
facets = None, pagenum = 1, pagelen = 20):
+ """
+ Perform query.
+ Temporary fixes:
+ Whoosh 2.4 raises an error when simultaneously using filters and facets
+ in search:
+ AttributeError: 'FacetCollector' object has no attribute 'offset'
+ The problem should be fixed in the next release. For more info read
+ https://bitbucket.org/mchaput/whoosh/issue/274
+ A workaround is introduced to join query and filter in query and not
+ using filter parameter. Remove the workaround when the fixed version
+ of Whoosh is applied.
+ """
with self.index.searcher() as searcher:
- parser = QueryParser("content", self.index.schema)
- if isinstance(query, basestring):
- query = unicode(query)
- parsed_query = parser.parse(unicode(query))
- else:
- parsed_query = query
-
sortedby = self._prepare_sortedby(sort)
- groupedby = self._prepare_groupedby(facets)
- self.env.log.debug("Whoosh query to execute: %s, sortedby = %s, \
- pagenum=%s, pagelen=%s, facets=%s",
- parsed_query,
- sortedby,
- pagenum,
- pagelen,
- groupedby,
- )
- raw_page = searcher.search_page(
- parsed_query,
+ groupedby = facets
+ query_filter = self._prepare_filter(filter)
+
+ #workaround of Whoosh bug, read method __doc__
+ query = self._workaround_join_query_and_filter(
+ query,
+ query_filter)
+
+ search_parameters = dict(
+ query = query,
pagenum = pagenum,
pagelen = pagelen,
sortedby = sortedby,
groupedby = groupedby,
+ maptype=sorting.Count,
+ #workaround of Whoosh bug, read method __doc__
+ #filter = query_filter,
)
-# raw_page = ResultsPage(whoosh_results, pagenum, pagelen)
- results = self._process_results(raw_page, fields)
+ self.env.log.debug("Whoosh query to execute: %s",
+ search_parameters)
+ raw_page = searcher.search_page(**search_parameters)
+ results = self._process_results(raw_page, fields, search_parameters)
return results
- def delete_doc(self, doc, commit=True):
- pass
-
- def commit(self):
- pass
-
- def optimize(self):
- pass
+ def _workaround_join_query_and_filter(
+ self,
+ query_expression,
+ query_filter):
+ if not query_filter:
+ return query_expression
+ return query.And((query_expression, query_filter))
- def recreate_index(self):
- self.index = self._create_index()
+ def _prepare_filter(self, filters):
+ if not filters:
+ return None
+ and_filters = []
+ for filter in filters:
+ and_filters.append(query.Term(
+ unicode(filter[0]),
+ unicode(filter[1])))
+ return query.And(and_filters)
- def open_or_create_index_if_missing(self):
- if index.exists_in(self.index_dir):
- self.index = index.open_dir(self.index_dir)
- else:
- self.index = self._create_index()
+ def _create_unique_id(self, type, id):
+ return u"%s:%s" % (type, id)
def _to_whoosh_format(self, value):
if isinstance(value, basestring):
@@ -167,13 +225,13 @@ class WhooshBackend(Component):
value = utc.localize(value)
return value
- def _prepare_groupedby(self, facets):
- if not facets:
- return None
- groupedby = sorting.Facets()
- for facet_name in facets:
- groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count)
- return groupedby
+# def _prepare_groupedby(self, facets):
+# if not facets:
+# return None
+# groupedby = sorting.Facets()
+# for facet_name in facets:
+# groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count)
+# return groupedby
def _prepare_sortedby(self, sort):
if not sort:
@@ -194,7 +252,7 @@ class WhooshBackend(Component):
def _is_desc(self, order):
return (order.lower()==DESC)
- def _process_results(self, page, fields):
+ def _process_results(self, page, fields, search_parameters = None):
# It's important to grab the hits first before slicing. Otherwise, this
# can cause pagination failures.
"""
@@ -213,6 +271,7 @@ class WhooshBackend(Component):
result_doc = self._process_record(fields, retrieved_record)
docs.append(result_doc)
results.docs = docs
+ results.debug["search_parameters"] = search_parameters
return results
def _process_record(self, fields, retrieved_record):
@@ -244,12 +303,7 @@ class WhooshBackend(Component):
facets_result[name] = non_paged_results.groups(name)
return facets_result
- def _create_index(self):
- self.log.info('Creating Whoosh index in %s' % self.index_dir)
- self._mkdir_if_not_exists()
- return index.create_in(self.index_dir, schema=self.SCHEMA)
-
- def _mkdir_if_not_exists(self):
+ def _make_dir_if_not_exists(self):
if not os.path.exists(self.index_dir):
os.mkdir(self.index_dir)
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/setup.py
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/setup.py?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/setup.py (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_search/setup.py Tue Jan 15 09:32:53 2013
@@ -29,7 +29,7 @@ Add free text search and query functiona
"""
versions = [
- (0, 4, 1),
+ (0, 4, 0),
]
latest = '.'.join(str(x) for x in versions[-1])
@@ -124,6 +124,7 @@ ENTRY_POINTS = {
'bhsearch.api = bhsearch.api',
'bhsearch.admin = bhsearch.admin',
'bhsearch.ticket_search = bhsearch.ticket_search',
+ 'bhsearch.wiki_search = bhsearch.wiki_search',
'bhsearch.query_parser = bhsearch.query_parser',
'bhsearch.whoosh_backend = bhsearch.whoosh_backend',
],
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/htdocs/bloodhound.css
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/htdocs/bloodhound.css?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/htdocs/bloodhound.css (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/htdocs/bloodhound.css Tue Jan 15 09:32:53 2013
@@ -218,7 +218,7 @@ pre.wiki {
padding-left: 5px;
}
-.ticket .properties .enum h5 {
+.ticket .properties .enum h5, #basic-properties h5 {
margin-top: 0px;
}
@@ -231,6 +231,13 @@ pre.wiki {
width: auto;
}
+.trac-loading {
+ background: url(../common/loading.gif) 0 50% no-repeat;
+ margin: 0 1em;
+ padding-left: 16px;
+ display: none;
+}
+
/* @end */
/* @group Quick Ticket fieldset */
@@ -289,7 +296,7 @@ h1, h2, h3, h4 {
white-space: nowrap;
}
-.clip.edit-active, .affix .clip-affic.edit-active {
+.clip.edit-active, .affix .clip-affix.edit-active {
overflow: visible;
}
/* @end */
@@ -368,6 +375,14 @@ h1, h2, h3, h4 {
/* @end */
+/* @group Admin */
+
+.table .default {
+ text-align: center
+}
+
+/* @end */
+
/* @group Reports */
.report span.foldable {
Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_enums.html
URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_enums.html?rev=1433322&r1=1433321&r2=1433322&view=diff
==============================================================================
--- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_enums.html (original)
+++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_theme/bhtheme/templates/bh_admin_enums.html Tue Jan 15 09:32:53 2013
@@ -111,7 +111,7 @@
<div class="span3">
<form class="well" id="addenum" method="post" action="">
<fieldset>
- <legend i18n:msg="label_singular">Add $label_singular</legend>
+ <legend i18n:msg="label_singular">Add $label_singular:</legend>
<label class="control-label" for="name">Name:</label>
<input class="input-medium" type="text" name="name" id="name"/>
<input class="btn" type="submit" name="add"