You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@bloodhound.apache.org by Antonia Horincar <an...@gmail.com> on 2014/06/20 02:45:21 UTC
Re: svn commit: r1604058 - in
/bloodhound/branches/bep_0014_solr/bloodhound_solr: bhsolr/admin.py
bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py bhsolr/templates/
bhsolr/web_ui.py setup.py
Created Sunburnt query chains for querying Solr; this is the equivalent of the DefaultQueryParser class that creates queries for Whoosh. Also implemented the IAdminCommandProvider interface for providing two trac-admin commands to pre-poluate BH with existing resources, and for optimising the index. However, the two trac-admin commands doesn’t seem to work for the moment (when trying to run them, I get a “command not found” error), but I’m currently trying to find out why this happens.
On 20 June 2014 at 01:40:08, ahorincar@apache.org (ahorincar@apache.org) wrote:
Author: ahorincar
Date: Fri Jun 20 00:39:42 2014
New Revision: 1604058
URL: http://svn.apache.org/r1604058
Log:
Added methods to create Sunburnt query chains
Added:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
Removed:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/htdocs/
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py
Modified:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py?rev=1604058&view=auto
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py (added)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py Fri Jun 20 00:39:42 2014
@@ -0,0 +1,16 @@
+r"""Administration commands for Bloodhound Solr Search."""
+from trac.core import Component, implements
+from trac.admin import IAdminCommandProvider
+from bhsolr.api import BloodhoundSolrApi
+
+class BloodhoundSolrSearchAdmin(Component):
+ """Bloodhound Solr Search administration component."""
+ implements(IAdminCommandProvider)
+
+ # IAdminCommandProvider methods
+ def get_admin_commands(self):
+ yield ('bhsolr populate_index', '', 'Populate Solr search index',
+ None, BloodhoundSolrApi(self.env).populate_index)
+ yield ('bhsolr optimize', '', 'Optimize Solr search index',
+ None, BloodhoundSearchApi(self.env).optimize)
+
Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py?rev=1604058&view=auto
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py (added)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py Fri Jun 20 00:39:42 2014
@@ -0,0 +1,7 @@
+from bhsolr.search_resources import TicketSearchModel
+from trac.core import Component, implements
+
+class BloodhoundSolrApi(Component):
+ def populate_index(self):
+ tickets = TicketSearchModel(self.env).get_entries_for_index()
+ print tickets
Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py?rev=1604058&r1=1604057&r2=1604058&view=diff
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py (original)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py Fri Jun 20 00:39:42 2014
@@ -7,17 +7,19 @@ from trac.ticket.api import TicketSystem
from bhsearch.search_resources.ticket_search import TicketIndexer
from datetime import datetime
from trac.util.datefmt import utc
+from bhsearch.api import ISearchBackend
+from bhsearch.query_parser import DefaultQueryParser
UNIQUE_ID = "unique_id"
-class SolrModel(Component):
+class SolrBackend(Component):
implements(ISearchBackend)
def __init__(self):
resource_filename = pkg_resources.resource_filename
path = resource_filename(__name__, "schemadoc")
- # self.solr_interface = Solr("http://localhost:8983/solr/", path + '/schema.xml').solr_interface
- self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface
+ self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface # TODO: Refactor
+ self.field_boosts = DefaultQueryParser(self.env).field_boosts
def add_doc(self, doc, operation_context=None):
@@ -34,13 +36,44 @@ class SolrModel(Component):
self.solr_interface.delete(unique_id)
- def optimize():
+ def optimize(self):
self.solr_interface.optimize()
+ def query(self, query, query_string, sort = None, fields = None, filter = None,
+ facets = None, pagenum = 1, pagelen = 20, highlight = False,
+ highlight_fields = None, context = None):
+
+ tokens = set([token.text for token in query.all_tokens()])
+ final_query_chain = None
+ for token in tokens:
+ token_query_chain = self._search_fields_for_token(token)
+ if final_query_chain == None:
+ final_query_chain = token_query_chain
+ else:
+ final_query_chain |= token_query_chain
+
+ print final_query_chain
+ return None
- def query(self, query):
- self.solr_instance.solr_interface.query(query).execute()
+ def is_index_outdated(self):
+ return False
+ def recreate_index(self):
+ return True
+
+ def start_operation(self):
+ return None
+
+ def _search_fields_for_token(self, token):
+ query_chain = None
+ for field, boost in self.field_boosts.iteritems():
+ field_token_dict = {field: token}
+ if query_chain == None:
+ query_chain = self.solr_interface.Q(**field_token_dict)**boost
+ else:
+ query_chain |= self.solr_interface.Q(**field_token_dict)**boost
+
+ return query_chain
def _reformat_doc(self, doc):
for key, value in doc.items():
@@ -75,12 +108,7 @@ class SolrModel(Component):
else:
return u"%s:%s" % (doc_type, doc_id)
-if __name__ == '__main__':
- env = trac.env.Environment("/Users/antonia/Documents/Code/bloodhound/installer/bloodhound/environments/main")
- db_connection = env.get_db_cnx()
- cursor = db_connection.cursor()
- a = cursor.execute("select * from ticket")
- ticket = a.fetchall()[0]
+ def getInstance(self):
+ return self.solr_interface
+
- # for result in si.query(name="Ticket").execute():
- # print result
Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py?rev=1604058&r1=1604057&r2=1604058&view=diff
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py (original)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Fri Jun 20 00:39:42 2014
@@ -1,6 +1,6 @@
from setuptools import setup, find_packages
-PKG_INFO = {'bhsolr': ['htdocs/*.*', 'templates/*', 'schemadoc/*.xml'],
+PKG_INFO = {'bhsolr': ['schemadoc/*.xml'],
'bhsolr.search_resources' : [],
}
@@ -8,9 +8,9 @@ PKG_INFO = {'bhsolr': ['htdocs/*.*', 'te
ENTRY_POINTS = {
'trac.plugins': [
- 'bhsolr.web_ui = bhsolr.web_ui',
'bhsolr.api = bhsolr.api',
'bhsolr.solr = bhsolr.solr',
+ 'bhsolr.admin = bhsolr.admin',
'bhsolr.solr_backend = bhsolr.solr_backend',
'bhsolr.search_resources.ticket_search = bhsolr.search_resources.ticket_search',
'bhsolr.search_resources.milestone_search = bhsolr.search_resources.milestone_search',
Re: svn commit: r1604058 - in
/bloodhound/branches/bep_0014_solr/bloodhound_solr: bhsolr/admin.py
bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py bhsolr/templates/
bhsolr/web_ui.py setup.py
Posted by Antonia Horincar <an...@gmail.com>.
On 30 June 2014 at 09:28:47, Andrej Golcov (andrej@digiverse.si) wrote:
Hi Antonia,
> Also, you might have noticed I called the paginate(rows=20000) method on the query chain (in the bhsolr.solr_backend query() method). By default, Solr fetches only 10 documents when a query has been made. And there is no way for fetching all query results [1]. I was thinking of a reasonable solution, but I would like your opinion on this matter:
>
> So, a solution for this would be to specify a maximum number of results to be retrieved. And the number should be as close as possible to the number of documents stored in the index. So I could keep track of how many documents were added to the index, and update the ‘rows’ parameter every time the number changes.
> Another solution would be to make multiple smaller queries (with the ‘rows’ parameter set to the max number of results per page) until all results have been fetched.
> How should I proceed in implementing this?
Why would do you want to get all found results from Solr? That could
be a real performance issue.
If you need only count, Solr returns found items count in its
response: https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results
Then I guess my second suggestion is more suitable, isn’t it? The Whoosh implementation follows this pattern as well, from what I saw.
Cheers, Andrej
Re: svn commit: r1604058 - in /bloodhound/branches/bep_0014_solr/bloodhound_solr:
bhsolr/admin.py bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py
bhsolr/templates/ bhsolr/web_ui.py setup.py
Posted by Andrej Golcov <an...@digiverse.si>.
Hi Antonia,
> Also, you might have noticed I called the paginate(rows=20000) method on the query chain (in the bhsolr.solr_backend query() method). By default, Solr fetches only 10 documents when a query has been made. And there is no way for fetching all query results [1]. I was thinking of a reasonable solution, but I would like your opinion on this matter:
>
> So, a solution for this would be to specify a maximum number of results to be retrieved. And the number should be as close as possible to the number of documents stored in the index. So I could keep track of how many documents were added to the index, and update the ‘rows’ parameter every time the number changes.
> Another solution would be to make multiple smaller queries (with the ‘rows’ parameter set to the max number of results per page) until all results have been fetched.
> How should I proceed in implementing this?
Why would do you want to get all found results from Solr? That could
be a real performance issue.
If you need only count, Solr returns found items count in its
response: https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results
Cheers, Andrej
Re: svn commit: r1604058 - in
/bloodhound/branches/bep_0014_solr/bloodhound_solr: bhsolr/admin.py
bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py bhsolr/templates/
bhsolr/web_ui.py setup.py
Posted by Antonia Horincar <an...@gmail.com>.
On 23 June 2014 at 08:54:57, Anže Starič (anze.staric@gmail.com) wrote:
On Mon, Jun 23, 2014 at 2:12 AM, Antonia Horincar
<an...@gmail.com> wrote:
> How about the case when the admin switches from one search backend to another, shouldn’t the appropriate index be populated with all existing resources in BH? This is mainly why I was thinking I need to implement my own admin commands.
This can also be done using bhsearch admin command. When admin decides
to switch search backends, he needs to modify search_backend setting
in trac.ini and run trac-admin bhsearch upgrade. This looks like a
reasonable workflow to me. We could extend the
environment_needs_upgrade method in BlodhoundSearchApi to monitor for
backend change and request an environment upgrade when it does, but I
do not think that this is a priority.
I didn’t know that we could achieve this using bhsearch upgrade, that’s why I thought I needed to implement another admin command. But that worked perfectly.
> I am currently working on displaying the retrieved Solr results in the interface. The results are currently shown in the interface, but I am working on applying highlighting and faceting.
>
> Also, I have a question regarding the meta keyword parsers. How are the DocTypeMetaKeywordParser and the other keyword parsers from bhsearch.query_parser used?
MetaKeywordParsers are just match_and_replace rules for words
beginning with a $. They are used in MetaKeywordPlugin which could be
summarized as: find all words that begin with a $ using a regexp
match, pass the word to MetaKeywordParsers, if any of them knows the
keyword it will return some text, which you use to replace the keyword
string in the original query.
> I understood in general what the DefaultQueryParser does, however I’m not sure I get how parser plugins are used in Whoosh. I would like to understand the query_parser module better because I used the DefaultQueryParser for parsing the query. I’m not sure if this is a good idea because basically it uses Whoosh for parsing the query, but it was easier for the moment. Should I try to implement my own query parser for Solr?
If I understand correctly, solrs expects the query as a string, which
it then parses internally. If it is not too hard to reconstruct the
query from whoosh, I would use the existing query parser, so you can
reuse the existing security processing and meta keyword parsing.
It’s actually easier to reconstruct the query from whoosh (by accessing attributes of query objects created by Whoosh), because otherwise I would have to implement a parser to correctly parse a raw query, which in my opinion is much more difficult to achieve.
If you want to know more about how whoosh parses queries, here is a
short description. Whoosh parses queries with a bunch of match and
filter plugins. Match plugins try to match the word to a predefined
regular expression and emit a node class upon match. Filters then
modify the generated list of nodes to group nodes based on operator
priority, manage terms without defined fields etc.
MetaKeywordPlugin is both a matcher and a filter. It matches all words
starting with a $ and passes them to MetaKeywordParsers. If a
MetaKeywordParsers understands a keyword, it expands it into a new
string ($ticket -> type:ticket), which is again parsed by whoosh.
Parsed representation of the expanded meta keyword is stored inside a
MetaKeywordNode. In the filter phase, MetaKeywordPlugin "flattens" the
meta keywords (replaces meta keyword nodes with the parsed
representation of the expanded text.
Thanks for the description, this really helped me understand more about the Whoosh query parser.
Anze
[1] https://github.com/apache/bloodhound/blob/trunk/bloodhound_search/bhsearch/api.py#L402
I am currently working on adding a “More like this” feature. At first, I was thinking of automatically displaying similar query results when a query has been made. But due to some limitations of Sunburnt, this would mean making two different requests to Solr (one for getting the query results, and one for getting the results that are similar to the initially retrieved results). Would it be better to have a “More like this” button next to the query results, and if a user chooses to see similar results, then a new request would be made to Solr? I began to implement the ITemplateStreamFilter interface for adding a button on the search results page, but haven’t successfully finished yet.
Also, you might have noticed I called the paginate(rows=20000) method on the query chain (in the bhsolr.solr_backend query() method). By default, Solr fetches only 10 documents when a query has been made. And there is no way for fetching all query results [1]. I was thinking of a reasonable solution, but I would like your opinion on this matter:
So, a solution for this would be to specify a maximum number of results to be retrieved. And the number should be as close as possible to the number of documents stored in the index. So I could keep track of how many documents were added to the index, and update the ‘rows’ parameter every time the number changes.
Another solution would be to make multiple smaller queries (with the ‘rows’ parameter set to the max number of results per page) until all results have been fetched.
How should I proceed in implementing this?
Thanks,
Antonia
[1] https://issues.apache.org/jira/browse/SOLR-534
Re: svn commit: r1604058 - in /bloodhound/branches/bep_0014_solr/bloodhound_solr:
bhsolr/admin.py bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py
bhsolr/templates/ bhsolr/web_ui.py setup.py
Posted by Anže Starič <an...@gmail.com>.
On Mon, Jun 23, 2014 at 2:12 AM, Antonia Horincar
<an...@gmail.com> wrote:
> How about the case when the admin switches from one search backend to another, shouldn’t the appropriate index be populated with all existing resources in BH? This is mainly why I was thinking I need to implement my own admin commands.
This can also be done using bhsearch admin command. When admin decides
to switch search backends, he needs to modify search_backend setting
in trac.ini and run trac-admin bhsearch upgrade. This looks like a
reasonable workflow to me. We could extend the
environment_needs_upgrade method in BlodhoundSearchApi to monitor for
backend change and request an environment upgrade when it does, but I
do not think that this is a priority.
> I am currently working on displaying the retrieved Solr results in the interface. The results are currently shown in the interface, but I am working on applying highlighting and faceting.
>
> Also, I have a question regarding the meta keyword parsers. How are the DocTypeMetaKeywordParser and the other keyword parsers from bhsearch.query_parser used?
MetaKeywordParsers are just match_and_replace rules for words
beginning with a $. They are used in MetaKeywordPlugin which could be
summarized as: find all words that begin with a $ using a regexp
match, pass the word to MetaKeywordParsers, if any of them knows the
keyword it will return some text, which you use to replace the keyword
string in the original query.
> I understood in general what the DefaultQueryParser does, however I’m not sure I get how parser plugins are used in Whoosh. I would like to understand the query_parser module better because I used the DefaultQueryParser for parsing the query. I’m not sure if this is a good idea because basically it uses Whoosh for parsing the query, but it was easier for the moment. Should I try to implement my own query parser for Solr?
If I understand correctly, solrs expects the query as a string, which
it then parses internally. If it is not too hard to reconstruct the
query from whoosh, I would use the existing query parser, so you can
reuse the existing security processing and meta keyword parsing.
If you want to know more about how whoosh parses queries, here is a
short description. Whoosh parses queries with a bunch of match and
filter plugins. Match plugins try to match the word to a predefined
regular expression and emit a node class upon match. Filters then
modify the generated list of nodes to group nodes based on operator
priority, manage terms without defined fields etc.
MetaKeywordPlugin is both a matcher and a filter. It matches all words
starting with a $ and passes them to MetaKeywordParsers. If a
MetaKeywordParsers understands a keyword, it expands it into a new
string ($ticket -> type:ticket), which is again parsed by whoosh.
Parsed representation of the expanded meta keyword is stored inside a
MetaKeywordNode. In the filter phase, MetaKeywordPlugin "flattens" the
meta keywords (replaces meta keyword nodes with the parsed
representation of the expanded text.
Anze
[1] https://github.com/apache/bloodhound/blob/trunk/bloodhound_search/bhsearch/api.py#L402
Re: svn commit: r1604058 - in
/bloodhound/branches/bep_0014_solr/bloodhound_solr: bhsolr/admin.py
bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py bhsolr/templates/
bhsolr/web_ui.py setup.py
Posted by Antonia Horincar <an...@gmail.com>.
On 20 June 2014 at 16:47:20, Anže Starič (anze.staric@gmail.com) wrote:
You do not need to implement your own admin commands, bhsearch
optimize_index should work.
How about the case when the admin switches from one search backend to another, shouldn’t the appropriate index be populated with all existing resources in BH? This is mainly why I was thinking I need to implement my own admin commands.
I took a look at your code, and the indexing part seems to be working.
I had to change the definition of start_operation [1] so it can be
used with the with statement, but the rest worked out of the box. The
code is used is pushed to my branch on github [2], if you want to take
a look. I used it with the following additions to the trac.ini:
[bhsearch]
search_backend = SolrBackend
solr_server_url = http://localhost:8983/solr/
[components]
bhsearch.query_suggestion.* = disabled
bhsearch.security.* = disabled
Thank you for the link, I took your example and added the config option for providing a Solr server path (I was planning on making this configurable, and not hard-coded, but I wasn’t sure of how to do this). I also made the changes you mentioned.
I am currently working on displaying the retrieved Solr results in the interface. The results are currently shown in the interface, but I am working on applying highlighting and faceting.
Also, I have a question regarding the meta keyword parsers. How are the DocTypeMetaKeywordParser and the other keyword parsers from bhsearch.query_parser used? I understood in general what the DefaultQueryParser does, however I’m not sure I get how parser plugins are used in Whoosh. I would like to understand the query_parser module better because I used the DefaultQueryParser for parsing the query. I’m not sure if this is a good idea because basically it uses Whoosh for parsing the query, but it was easier for the moment. Should I try to implement my own query parser for Solr?
Best,
Antonia
Anze
[1] Changes to start_operation:
change
def start_operation(self):
return None
to
@contextmanager
def start_operation(self):
yield
[2] https://github.com/astaric/bloodhound/commit/27db226d8e03ffb1c83e5a906b73bfdfe8dc1c25
On Fri, Jun 20, 2014 at 2:45 AM, Antonia Horincar
<an...@gmail.com> wrote:
> Created Sunburnt query chains for querying Solr; this is the equivalent of the DefaultQueryParser class that creates queries for Whoosh. Also implemented the IAdminCommandProvider interface for providing two trac-admin commands to pre-poluate BH with existing resources, and for optimising the index. However, the two trac-admin commands doesn’t seem to work for the moment (when trying to run them, I get a “command not found” error), but I’m currently trying to find out why this happens.
>
> On 20 June 2014 at 01:40:08, ahorincar@apache.org (ahorincar@apache.org) wrote:
>
> Author: ahorincar
> Date: Fri Jun 20 00:39:42 2014
> New Revision: 1604058
>
> URL: http://svn.apache.org/r1604058
> Log:
> Added methods to create Sunburnt query chains
>
> Added:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
> Removed:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/htdocs/
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py
> Modified:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
> bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
>
> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py?rev=1604058&view=auto
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py (added)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py Fri Jun 20 00:39:42 2014
> @@ -0,0 +1,16 @@
> +r"""Administration commands for Bloodhound Solr Search."""
> +from trac.core import Component, implements
> +from trac.admin import IAdminCommandProvider
> +from bhsolr.api import BloodhoundSolrApi
> +
> +class BloodhoundSolrSearchAdmin(Component):
> + """Bloodhound Solr Search administration component."""
> + implements(IAdminCommandProvider)
> +
> + # IAdminCommandProvider methods
> + def get_admin_commands(self):
> + yield ('bhsolr populate_index', '', 'Populate Solr search index',
> + None, BloodhoundSolrApi(self.env).populate_index)
> + yield ('bhsolr optimize', '', 'Optimize Solr search index',
> + None, BloodhoundSearchApi(self.env).optimize)
> +
>
> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py?rev=1604058&view=auto
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py (added)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py Fri Jun 20 00:39:42 2014
> @@ -0,0 +1,7 @@
> +from bhsolr.search_resources import TicketSearchModel
> +from trac.core import Component, implements
> +
> +class BloodhoundSolrApi(Component):
> + def populate_index(self):
> + tickets = TicketSearchModel(self.env).get_entries_for_index()
> + print tickets
>
> Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py?rev=1604058&r1=1604057&r2=1604058&view=diff
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py (original)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py Fri Jun 20 00:39:42 2014
> @@ -7,17 +7,19 @@ from trac.ticket.api import TicketSystem
> from bhsearch.search_resources.ticket_search import TicketIndexer
> from datetime import datetime
> from trac.util.datefmt import utc
> +from bhsearch.api import ISearchBackend
> +from bhsearch.query_parser import DefaultQueryParser
>
> UNIQUE_ID = "unique_id"
>
> -class SolrModel(Component):
> +class SolrBackend(Component):
> implements(ISearchBackend)
>
> def __init__(self):
> resource_filename = pkg_resources.resource_filename
> path = resource_filename(__name__, "schemadoc")
> - # self.solr_interface = Solr("http://localhost:8983/solr/", path + '/schema.xml').solr_interface
> - self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface
> + self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface # TODO: Refactor
> + self.field_boosts = DefaultQueryParser(self.env).field_boosts
>
>
> def add_doc(self, doc, operation_context=None):
> @@ -34,13 +36,44 @@ class SolrModel(Component):
> self.solr_interface.delete(unique_id)
>
>
> - def optimize():
> + def optimize(self):
> self.solr_interface.optimize()
>
> + def query(self, query, query_string, sort = None, fields = None, filter = None,
> + facets = None, pagenum = 1, pagelen = 20, highlight = False,
> + highlight_fields = None, context = None):
> +
> + tokens = set([token.text for token in query.all_tokens()])
> + final_query_chain = None
> + for token in tokens:
> + token_query_chain = self._search_fields_for_token(token)
> + if final_query_chain == None:
> + final_query_chain = token_query_chain
> + else:
> + final_query_chain |= token_query_chain
> +
> + print final_query_chain
> + return None
>
> - def query(self, query):
> - self.solr_instance.solr_interface.query(query).execute()
> + def is_index_outdated(self):
> + return False
>
> + def recreate_index(self):
> + return True
> +
> + def start_operation(self):
> + return None
> +
> + def _search_fields_for_token(self, token):
> + query_chain = None
> + for field, boost in self.field_boosts.iteritems():
> + field_token_dict = {field: token}
> + if query_chain == None:
> + query_chain = self.solr_interface.Q(**field_token_dict)**boost
> + else:
> + query_chain |= self.solr_interface.Q(**field_token_dict)**boost
> +
> + return query_chain
>
> def _reformat_doc(self, doc):
> for key, value in doc.items():
> @@ -75,12 +108,7 @@ class SolrModel(Component):
> else:
> return u"%s:%s" % (doc_type, doc_id)
>
> -if __name__ == '__main__':
> - env = trac.env.Environment("/Users/antonia/Documents/Code/bloodhound/installer/bloodhound/environments/main")
> - db_connection = env.get_db_cnx()
> - cursor = db_connection.cursor()
> - a = cursor.execute("select * from ticket")
> - ticket = a.fetchall()[0]
> + def getInstance(self):
> + return self.solr_interface
> +
>
> - # for result in si.query(name="Ticket").execute():
> - # print result
>
> Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py?rev=1604058&r1=1604057&r2=1604058&view=diff
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py (original)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Fri Jun 20 00:39:42 2014
> @@ -1,6 +1,6 @@
> from setuptools import setup, find_packages
>
> -PKG_INFO = {'bhsolr': ['htdocs/*.*', 'templates/*', 'schemadoc/*.xml'],
> +PKG_INFO = {'bhsolr': ['schemadoc/*.xml'],
> 'bhsolr.search_resources' : [],
> }
>
> @@ -8,9 +8,9 @@ PKG_INFO = {'bhsolr': ['htdocs/*.*', 'te
>
> ENTRY_POINTS = {
> 'trac.plugins': [
> - 'bhsolr.web_ui = bhsolr.web_ui',
> 'bhsolr.api = bhsolr.api',
> 'bhsolr.solr = bhsolr.solr',
> + 'bhsolr.admin = bhsolr.admin',
> 'bhsolr.solr_backend = bhsolr.solr_backend',
> 'bhsolr.search_resources.ticket_search = bhsolr.search_resources.ticket_search',
> 'bhsolr.search_resources.milestone_search = bhsolr.search_resources.milestone_search',
>
>
Re: svn commit: r1604058 - in /bloodhound/branches/bep_0014_solr/bloodhound_solr:
bhsolr/admin.py bhsolr/api.py bhsolr/htdocs/ bhsolr/solr_backend.py
bhsolr/templates/ bhsolr/web_ui.py setup.py
Posted by Anže Starič <an...@gmail.com>.
You do not need to implement your own admin commands, bhsearch
optimize_index should work.
I took a look at your code, and the indexing part seems to be working.
I had to change the definition of start_operation [1] so it can be
used with the with statement, but the rest worked out of the box. The
code is used is pushed to my branch on github [2], if you want to take
a look. I used it with the following additions to the trac.ini:
[bhsearch]
search_backend = SolrBackend
solr_server_url = http://localhost:8983/solr/
[components]
bhsearch.query_suggestion.* = disabled
bhsearch.security.* = disabled
Anze
[1] Changes to start_operation:
change
def start_operation(self):
return None
to
@contextmanager
def start_operation(self):
yield
[2] https://github.com/astaric/bloodhound/commit/27db226d8e03ffb1c83e5a906b73bfdfe8dc1c25
On Fri, Jun 20, 2014 at 2:45 AM, Antonia Horincar
<an...@gmail.com> wrote:
> Created Sunburnt query chains for querying Solr; this is the equivalent of the DefaultQueryParser class that creates queries for Whoosh. Also implemented the IAdminCommandProvider interface for providing two trac-admin commands to pre-poluate BH with existing resources, and for optimising the index. However, the two trac-admin commands doesn’t seem to work for the moment (when trying to run them, I get a “command not found” error), but I’m currently trying to find out why this happens.
>
> On 20 June 2014 at 01:40:08, ahorincar@apache.org (ahorincar@apache.org) wrote:
>
> Author: ahorincar
> Date: Fri Jun 20 00:39:42 2014
> New Revision: 1604058
>
> URL: http://svn.apache.org/r1604058
> Log:
> Added methods to create Sunburnt query chains
>
> Added:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
> Removed:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/htdocs/
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py
> Modified:
> bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
> bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
>
> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py?rev=1604058&view=auto
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py (added)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py Fri Jun 20 00:39:42 2014
> @@ -0,0 +1,16 @@
> +r"""Administration commands for Bloodhound Solr Search."""
> +from trac.core import Component, implements
> +from trac.admin import IAdminCommandProvider
> +from bhsolr.api import BloodhoundSolrApi
> +
> +class BloodhoundSolrSearchAdmin(Component):
> + """Bloodhound Solr Search administration component."""
> + implements(IAdminCommandProvider)
> +
> + # IAdminCommandProvider methods
> + def get_admin_commands(self):
> + yield ('bhsolr populate_index', '', 'Populate Solr search index',
> + None, BloodhoundSolrApi(self.env).populate_index)
> + yield ('bhsolr optimize', '', 'Optimize Solr search index',
> + None, BloodhoundSearchApi(self.env).optimize)
> +
>
> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py?rev=1604058&view=auto
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py (added)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py Fri Jun 20 00:39:42 2014
> @@ -0,0 +1,7 @@
> +from bhsolr.search_resources import TicketSearchModel
> +from trac.core import Component, implements
> +
> +class BloodhoundSolrApi(Component):
> + def populate_index(self):
> + tickets = TicketSearchModel(self.env).get_entries_for_index()
> + print tickets
>
> Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py?rev=1604058&r1=1604057&r2=1604058&view=diff
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py (original)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py Fri Jun 20 00:39:42 2014
> @@ -7,17 +7,19 @@ from trac.ticket.api import TicketSystem
> from bhsearch.search_resources.ticket_search import TicketIndexer
> from datetime import datetime
> from trac.util.datefmt import utc
> +from bhsearch.api import ISearchBackend
> +from bhsearch.query_parser import DefaultQueryParser
>
> UNIQUE_ID = "unique_id"
>
> -class SolrModel(Component):
> +class SolrBackend(Component):
> implements(ISearchBackend)
>
> def __init__(self):
> resource_filename = pkg_resources.resource_filename
> path = resource_filename(__name__, "schemadoc")
> - # self.solr_interface = Solr("http://localhost:8983/solr/", path + '/schema.xml').solr_interface
> - self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface
> + self.solr_interface = Solr("http://localhost:8983/solr/").solr_interface # TODO: Refactor
> + self.field_boosts = DefaultQueryParser(self.env).field_boosts
>
>
> def add_doc(self, doc, operation_context=None):
> @@ -34,13 +36,44 @@ class SolrModel(Component):
> self.solr_interface.delete(unique_id)
>
>
> - def optimize():
> + def optimize(self):
> self.solr_interface.optimize()
>
> + def query(self, query, query_string, sort = None, fields = None, filter = None,
> + facets = None, pagenum = 1, pagelen = 20, highlight = False,
> + highlight_fields = None, context = None):
> +
> + tokens = set([token.text for token in query.all_tokens()])
> + final_query_chain = None
> + for token in tokens:
> + token_query_chain = self._search_fields_for_token(token)
> + if final_query_chain == None:
> + final_query_chain = token_query_chain
> + else:
> + final_query_chain |= token_query_chain
> +
> + print final_query_chain
> + return None
>
> - def query(self, query):
> - self.solr_instance.solr_interface.query(query).execute()
> + def is_index_outdated(self):
> + return False
>
> + def recreate_index(self):
> + return True
> +
> + def start_operation(self):
> + return None
> +
> + def _search_fields_for_token(self, token):
> + query_chain = None
> + for field, boost in self.field_boosts.iteritems():
> + field_token_dict = {field: token}
> + if query_chain == None:
> + query_chain = self.solr_interface.Q(**field_token_dict)**boost
> + else:
> + query_chain |= self.solr_interface.Q(**field_token_dict)**boost
> +
> + return query_chain
>
> def _reformat_doc(self, doc):
> for key, value in doc.items():
> @@ -75,12 +108,7 @@ class SolrModel(Component):
> else:
> return u"%s:%s" % (doc_type, doc_id)
>
> -if __name__ == '__main__':
> - env = trac.env.Environment("/Users/antonia/Documents/Code/bloodhound/installer/bloodhound/environments/main")
> - db_connection = env.get_db_cnx()
> - cursor = db_connection.cursor()
> - a = cursor.execute("select * from ticket")
> - ticket = a.fetchall()[0]
> + def getInstance(self):
> + return self.solr_interface
> +
>
> - # for result in si.query(name="Ticket").execute():
> - # print result
>
> Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
> URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py?rev=1604058&r1=1604057&r2=1604058&view=diff
> ==============================================================================
> --- bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py (original)
> +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Fri Jun 20 00:39:42 2014
> @@ -1,6 +1,6 @@
> from setuptools import setup, find_packages
>
> -PKG_INFO = {'bhsolr': ['htdocs/*.*', 'templates/*', 'schemadoc/*.xml'],
> +PKG_INFO = {'bhsolr': ['schemadoc/*.xml'],
> 'bhsolr.search_resources' : [],
> }
>
> @@ -8,9 +8,9 @@ PKG_INFO = {'bhsolr': ['htdocs/*.*', 'te
>
> ENTRY_POINTS = {
> 'trac.plugins': [
> - 'bhsolr.web_ui = bhsolr.web_ui',
> 'bhsolr.api = bhsolr.api',
> 'bhsolr.solr = bhsolr.solr',
> + 'bhsolr.admin = bhsolr.admin',
> 'bhsolr.solr_backend = bhsolr.solr_backend',
> 'bhsolr.search_resources.ticket_search = bhsolr.search_resources.ticket_search',
> 'bhsolr.search_resources.milestone_search = bhsolr.search_resources.milestone_search',
>
>