You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ko...@apache.org on 2011/01/28 17:58:46 UTC
svn commit: r1064775 - in /lucene/dev/branches/branch_3x: ./ lucene/ solr/
solr/src/java/org/apache/solr/handler/
solr/src/java/org/apache/solr/handler/component/
solr/src/test/org/apache/solr/handler/
Author: koji
Date: Fri Jan 28 16:58:46 2011
New Revision: 1064775
URL: http://svn.apache.org/viewvc?rev=1064775&view=rev
Log:
SOLR-860: Add debug output for MoreLikeThis
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/CHANGES.txt
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java
Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1064775&r1=1064774&r2=1064775&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Fri Jan 28 16:58:46 2011
@@ -252,6 +252,7 @@ New Features
* SOLR-2263: Add ability for RawResponseWriter to stream binary files as well as
text files. (Eric Pugh via yonik)
+* SOLR-860: Add debug output for MoreLikeThis. (koji)
Optimizations
----------------------
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=1064775&r1=1064774&r2=1064775&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java Fri Jan 28 16:58:46 2011
@@ -193,24 +193,27 @@ public class MoreLikeThisHandler extends
rsp.add( "facet_counts", f.getFacetCounts() );
}
}
-
+
+ boolean dbg = req.getParams().getBool(CommonParams.DEBUG_QUERY, false);
// Copied from StandardRequestHandler... perhaps it should be added to doStandardDebug?
- try {
- NamedList<Object> dbg = SolrPluginUtils.doStandardDebug(req, q, mlt.mltquery, mltDocs.docList );
- if (null != dbg) {
- if (null != filters) {
- dbg.add("filter_queries",req.getParams().getParams(CommonParams.FQ));
- List<String> fqs = new ArrayList<String>(filters.size());
- for (Query fq : filters) {
- fqs.add(QueryParsing.toString(fq, req.getSchema()));
+ if (dbg == true) {
+ try {
+ NamedList<Object> dbgInfo = SolrPluginUtils.doStandardDebug(req, q, mlt.getRawMLTQuery(), mltDocs.docList);
+ if (null != dbgInfo) {
+ if (null != filters) {
+ dbgInfo.add("filter_queries",req.getParams().getParams(CommonParams.FQ));
+ List<String> fqs = new ArrayList<String>(filters.size());
+ for (Query fq : filters) {
+ fqs.add(QueryParsing.toString(fq, req.getSchema()));
+ }
+ dbgInfo.add("parsed_filter_queries",fqs);
}
- dbg.add("parsed_filter_queries",fqs);
+ rsp.add("debug", dbg);
}
- rsp.add("debug", dbg);
+ } catch (Exception e) {
+ SolrException.logOnce(SolrCore.log, "Exception during debug", e);
+ rsp.add("exception_during_debug", SolrException.toStr(e));
}
- } catch (Exception e) {
- SolrException.logOnce(SolrCore.log, "Exception during debug", e);
- rsp.add("exception_during_debug", SolrException.toStr(e));
}
}
@@ -242,8 +245,6 @@ public class MoreLikeThisHandler extends
final boolean needDocSet;
Map<String,Float> boostFields;
- Query mltquery; // expose this for debugging
-
public MoreLikeThisHelper( SolrParams params, SolrIndexSearcher searcher )
{
this.searcher = searcher;
@@ -273,9 +274,26 @@ public class MoreLikeThisHandler extends
boostFields = SolrPluginUtils.parseFieldBoosts(params.getParams(MoreLikeThisParams.QF));
}
- private void setBoosts(Query mltquery) {
+ private Query rawMLTQuery;
+ private Query boostedMLTQuery;
+ private BooleanQuery realMLTQuery;
+
+ public Query getRawMLTQuery(){
+ return rawMLTQuery;
+ }
+
+ public Query getBoostedMLTQuery(){
+ return boostedMLTQuery;
+ }
+
+ public Query getRealMLTQuery(){
+ return realMLTQuery;
+ }
+
+ private Query getBoostedQuery(Query mltquery) {
+ BooleanQuery boostedQuery = (BooleanQuery)mltquery.clone();
if (boostFields.size() > 0) {
- List clauses = ((BooleanQuery)mltquery).clauses();
+ List clauses = boostedQuery.clauses();
for( Object o : clauses ) {
TermQuery q = (TermQuery)((BooleanClause)o).getQuery();
Float b = this.boostFields.get(q.getTerm().field());
@@ -284,49 +302,51 @@ public class MoreLikeThisHandler extends
}
}
}
+ return boostedQuery;
}
public DocListAndSet getMoreLikeThis( int id, int start, int rows, List<Query> filters, List<InterestingTerm> terms, int flags ) throws IOException
{
Document doc = reader.document(id);
- mltquery = mlt.like(id);
- setBoosts(mltquery);
+ rawMLTQuery = mlt.like(id);
+ boostedMLTQuery = getBoostedQuery( rawMLTQuery );
if( terms != null ) {
- fillInterestingTermsFromMLTQuery( mltquery, terms );
+ fillInterestingTermsFromMLTQuery( rawMLTQuery, terms );
}
// exclude current document from results
- BooleanQuery mltQuery = new BooleanQuery();
- mltQuery.add(mltquery, BooleanClause.Occur.MUST);
- mltQuery.add(
+ realMLTQuery = new BooleanQuery();
+ realMLTQuery.add(boostedMLTQuery, BooleanClause.Occur.MUST);
+ realMLTQuery.add(
new TermQuery(new Term(uniqueKeyField.getName(), uniqueKeyField.getType().storedToIndexed(doc.getFieldable(uniqueKeyField.getName())))),
BooleanClause.Occur.MUST_NOT);
DocListAndSet results = new DocListAndSet();
if (this.needDocSet) {
- results = searcher.getDocListAndSet(mltQuery, filters, null, start, rows, flags);
+ results = searcher.getDocListAndSet(realMLTQuery, filters, null, start, rows, flags);
} else {
- results.docList = searcher.getDocList(mltQuery, filters, null, start, rows, flags);
+ results.docList = searcher.getDocList(realMLTQuery, filters, null, start, rows, flags);
}
return results;
}
public DocListAndSet getMoreLikeThis( Reader reader, int start, int rows, List<Query> filters, List<InterestingTerm> terms, int flags ) throws IOException
{
- mltquery = mlt.like(reader);
- setBoosts(mltquery);
+ rawMLTQuery = mlt.like(reader);
+ boostedMLTQuery = getBoostedQuery( rawMLTQuery );
if( terms != null ) {
- fillInterestingTermsFromMLTQuery( mltquery, terms );
+ fillInterestingTermsFromMLTQuery( boostedMLTQuery, terms );
}
DocListAndSet results = new DocListAndSet();
if (this.needDocSet) {
- results = searcher.getDocListAndSet(mltquery, filters, null, start, rows, flags);
+ results = searcher.getDocListAndSet( boostedMLTQuery, filters, null, start, rows, flags);
} else {
- results.docList = searcher.getDocList(mltquery, filters, null, start, rows, flags);
+ results.docList = searcher.getDocList( boostedMLTQuery, filters, null, start, rows, flags);
}
return results;
}
-
+
+ @Deprecated
public NamedList<DocList> getMoreLikeThese( DocList docs, int rows, int flags ) throws IOException
{
IndexSchema schema = searcher.getSchema();
@@ -345,7 +365,7 @@ public class MoreLikeThisHandler extends
private void fillInterestingTermsFromMLTQuery( Query query, List<InterestingTerm> terms )
{
- List clauses = ((BooleanQuery)mltquery).clauses();
+ List clauses = ((BooleanQuery)query).clauses();
for( Object o : clauses ) {
TermQuery q = (TermQuery)((BooleanClause)o).getQuery();
InterestingTerm it = new InterestingTerm();
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java?rev=1064775&r1=1064774&r2=1064775&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java Fri Jan 28 16:58:46 2011
@@ -23,8 +23,12 @@ import java.net.URL;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.handler.MoreLikeThisHandler;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.SolrIndexSearcher;
/**
@@ -50,18 +54,59 @@ public class MoreLikeThisComponent exten
if( p.getBool( MoreLikeThisParams.MLT, false ) ) {
SolrIndexSearcher searcher = rb.req.getSearcher();
- MoreLikeThisHandler.MoreLikeThisHelper mlt
- = new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher );
-
- int mltcount = p.getInt( MoreLikeThisParams.DOC_COUNT, 5 );
- NamedList<DocList> sim = mlt.getMoreLikeThese(
- rb.getResults().docList, mltcount, rb.getFieldFlags() );
+ NamedList<DocList> sim = getMoreLikeThese( rb, searcher,
+ rb.getResults().docList, rb.getFieldFlags() );
// TODO ???? add this directly to the response?
rb.rsp.add( "moreLikeThis", sim );
}
}
+ NamedList<DocList> getMoreLikeThese( ResponseBuilder rb, SolrIndexSearcher searcher,
+ DocList docs, int flags ) throws IOException {
+ SolrParams p = rb.req.getParams();
+ IndexSchema schema = searcher.getSchema();
+ MoreLikeThisHandler.MoreLikeThisHelper mltHelper
+ = new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher );
+ NamedList<DocList> mlt = new SimpleOrderedMap<DocList>();
+ DocIterator iterator = docs.iterator();
+
+ SimpleOrderedMap<Object> dbg = null;
+ if( rb.isDebug() ){
+ dbg = new SimpleOrderedMap<Object>();
+ }
+
+ while( iterator.hasNext() ) {
+ int id = iterator.nextDoc();
+ int rows = p.getInt( MoreLikeThisParams.DOC_COUNT, 5 );
+ DocListAndSet sim = mltHelper.getMoreLikeThis( id, 0, rows, null, null, flags );
+ String name = schema.printableUniqueKey( searcher.doc( id ) );
+ mlt.add(name, sim.docList);
+
+ if( dbg != null ){
+ SimpleOrderedMap<Object> docDbg = new SimpleOrderedMap<Object>();
+ docDbg.add( "rawMLTQuery", mltHelper.getRawMLTQuery().toString() );
+ docDbg.add( "boostedMLTQuery", mltHelper.getBoostedMLTQuery().toString() );
+ docDbg.add( "realMLTQuery", mltHelper.getRealMLTQuery().toString() );
+ SimpleOrderedMap<Object> explains = new SimpleOrderedMap<Object>();
+ DocIterator mltIte = sim.docList.iterator();
+ while( mltIte.hasNext() ){
+ int mltid = mltIte.nextDoc();
+ String key = schema.printableUniqueKey( searcher.doc( mltid ) );
+ explains.add( key, searcher.explain( mltHelper.getRealMLTQuery(), mltid ) );
+ }
+ docDbg.add( "explain", explains );
+ dbg.add( name, docDbg );
+ }
+ }
+
+ // add debug information
+ if( dbg != null ){
+ rb.addDebugInfo( "moreLikeThis", dbg );
+ }
+ return mlt;
+ }
+
/////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
Modified: lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java?rev=1064775&r1=1064774&r2=1064775&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java Fri Jan 28 16:58:46 2011
@@ -102,8 +102,23 @@ public class MoreLikeThisHandlerTest ext
params.put(CommonParams.Q, new String[]{"id:44"});
assertQ("morelike this - harrison ford",mltreq
,"//result/doc[1]/int[@name='id'][.='45']");
-
- params.put(CommonParams.Q, new String[]{"id:42"});
+
+ // test MoreLikeThis debug
+ params.put(CommonParams.DEBUG_QUERY, new String[]{"true"});
+ assertQ("morelike this - harrison ford",mltreq
+ ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']"
+ ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']"
+ ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']"
+ ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']"
+ );
+
+ // test that qparser plugins work
+ params.remove(CommonParams.DEBUG_QUERY);
+ params.put(CommonParams.Q, new String[]{"{!field f=id}44"});
+ assertQ(mltreq
+ ,"//result/doc[1]/int[@name='id'][.='45']");
+
+ params.put(CommonParams.Q, new String[]{"id:42"});
params.put(MoreLikeThisParams.QF,new String[]{"name^5.0 subword^0.1"});
assertQ("morelikethis with weights",mltreq
,"//result/doc[1]/int[@name='id'][.='43']"