You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2008/02/26 20:47:10 UTC
svn commit: r631357 [1/2] - in /lucene/solr/trunk: ./
client/java/solrj/src/org/apache/solr/client/solrj/response/
src/java/org/apache/solr/common/util/ src/java/org/apache/solr/handler/
src/java/org/apache/solr/handler/component/ src/java/org/apache/s...
Author: yonik
Date: Tue Feb 26 11:47:07 2008
New Revision: 631357
URL: http://svn.apache.org/viewvc?rev=631357&view=rev
Log:
SOLR-303 Distributed Search over HTTP
Added:
lucene/solr/trunk/src/java/org/apache/solr/handler/component/ShardDoc.java (with props)
lucene/solr/trunk/src/java/org/apache/solr/handler/component/ShardRequest.java (with props)
lucene/solr/trunk/src/test/org/apache/solr/TestDistributedSearch.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/build.xml
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java
lucene/solr/trunk/src/java/org/apache/solr/common/util/StrUtils.java
lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java
lucene/solr/trunk/src/java/org/apache/solr/handler/StandardRequestHandler.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/DebugComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/FacetComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/HighlightComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchComponent.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java
lucene/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java
lucene/solr/trunk/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
lucene/solr/trunk/src/java/org/apache/solr/search/QParser.java
lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/solr/trunk/src/java/org/apache/solr/search/SortSpec.java
lucene/solr/trunk/src/java/org/apache/solr/search/Sorting.java
lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java
lucene/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java
lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Tue Feb 26 11:47:07 2008
@@ -199,6 +199,15 @@
with "304 Not Modified" when appropriate. New options have been added
to solrconfig.xml to influence this behavior.
(Thomas Peuss via hossman)
+
+40. SOLR-303: Distributed Search over HTTP. Specification of shards
+ argument causes Solr to query those shards and merge the results
+ into a single response. Querying, field faceting (sorted only),
+ query faceting, highlighting, and debug information are supported
+ in distributed mode.
+ (Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
+ ryan, yonik)
+
Changes in runtime behavior
Modified: lucene/solr/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/build.xml?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/build.xml (original)
+++ lucene/solr/trunk/build.xml Tue Feb 26 11:47:07 2008
@@ -235,9 +235,10 @@
depends="compile-common,init-forrest-entities">
<solr-javac destdir="${dest}/core"
- classpathref="compile.classpath">
+ classpathref="compile.classpath.solrj-embedded">
<src path="${src}/java" />
<src path="${src}/webapp/src" />
+ <src path="client/java/solrj/src" />
<exclude name="org/apache/solr/common/**" />
</solr-javac>
</target>
@@ -628,6 +629,7 @@
<include name="${fullnamever}.jar" />
<include name="${fullname}-common-${version}.jar" />
</lib>
+ <lib dir="client/java/solrj/lib"/>
<fileset dir="${src}/webapp/resources" />
<metainf dir="${basedir}" includes="LICENSE.txt,NOTICE.txt"/>
</war>
Modified: lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java (original)
+++ lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/QueryResponse.java Tue Feb 26 11:47:07 2008
@@ -51,7 +51,6 @@
// Debug Info
private Map<String,Object> _debugMap = null;
- private Map<String,Integer> _docIdMap = null;
private Map<String,String> _explainMap = null;
public QueryResponse( NamedList<Object> res )
@@ -81,9 +80,7 @@
}
}
}
-
- private static final String DKEY = ",internal_docid=";
-
+
private void extractDebugInfo( NamedList<Object> debug )
{
_debugMap = new LinkedHashMap<String, Object>(); // keep the order
@@ -93,20 +90,11 @@
// Parse out interisting bits from the debug info
_explainMap = new HashMap<String, String>();
- _docIdMap = new HashMap<String, Integer>();
NamedList<String> explain = (NamedList<String>)_debugMap.get( "explain" );
if( explain != null ) {
for( Map.Entry<String, String> info : explain ) {
String key = info.getKey();
- int idx0 = key.indexOf( '=' )+1;
- int idx1 = info.getKey().indexOf( DKEY );
- int idx2 = idx1 + DKEY.length();
-
- String id = key.substring( idx0, idx1 );
- String docID = key.substring( idx2 );
-
- _explainMap.put( id, info.getValue() );
- _docIdMap.put( id, Integer.valueOf( docID ) );
+ _explainMap.put( key, info.getValue() );
}
}
}
@@ -180,10 +168,6 @@
public Map<String, Object> getDebugMap() {
return _debugMap;
- }
-
- public Map<String, Integer> getDocIdMap() {
- return _docIdMap;
}
public Map<String, String> getExplainMap() {
Modified: lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java (original)
+++ lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/response/SolrResponseBase.java Tue Feb 26 11:47:07 2008
@@ -25,7 +25,7 @@
* @version $Id$
* @since solr 1.3
*/
-public abstract class SolrResponseBase extends SolrResponse
+public class SolrResponseBase extends SolrResponse
{
private long elapsedTime = -1;
private NamedList<Object> response = null;
Modified: lucene/solr/trunk/src/java/org/apache/solr/common/util/StrUtils.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/common/util/StrUtils.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/common/util/StrUtils.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/common/util/StrUtils.java Tue Feb 26 11:47:07 2008
@@ -118,6 +118,27 @@
return lst;
}
+ /** Creates a backslash escaped string, joining all the items. */
+ public static String join(List<String> items, char separator) {
+ StringBuilder sb = new StringBuilder(items.size() << 3);
+ boolean first=true;
+ for (String item : items) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(separator);
+ }
+ for (int i=0; i<item.length(); i++) {
+ char ch = item.charAt(i);
+ if (ch=='\\' || ch == separator) {
+ sb.append('\\');
+ }
+ sb.append(ch);
+ }
+ }
+ return sb.toString();
+ }
+
public static List<String> splitWS(String s, boolean decode) {
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/MoreLikeThisHandler.java Tue Feb 26 11:47:07 2008
@@ -179,7 +179,7 @@
rsp.add( "facet_counts", null );
}
else {
- SimpleFacets f = new SimpleFacets(searcher, mltDocs.docSet, params );
+ SimpleFacets f = new SimpleFacets(req, mltDocs.docSet, params );
rsp.add( "facet_counts", f.getFacetCounts() );
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java Tue Feb 26 11:47:07 2008
@@ -27,6 +27,7 @@
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.util.SolrPluginUtils;
+import org.apache.lucene.queryParser.ParseException;
import java.net.URL;
@@ -117,6 +118,9 @@
handleRequestBody( req, rsp );
} catch (Exception e) {
SolrException.log(SolrCore.log,e);
+ if (e instanceof ParseException) {
+ e = new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
rsp.setException(e);
numErrors++;
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/StandardRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/StandardRequestHandler.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/StandardRequestHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/StandardRequestHandler.java Tue Feb 26 11:47:07 2008
@@ -17,10 +17,11 @@
package org.apache.solr.handler;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.component.*;
+
import java.net.MalformedURLException;
import java.net.URL;
-
-import org.apache.solr.handler.component.SearchHandler;
/**
* @version $Id$
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/DebugComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/DebugComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/DebugComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/DebugComponent.java Tue Feb 26 11:47:07 2008
@@ -18,21 +18,20 @@
package org.apache.solr.handler.component;
import static org.apache.solr.common.params.CommonParams.FQ;
+import org.apache.solr.common.params.HighlightParams;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
import org.apache.lucene.search.Query;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.util.SolrPluginUtils;
/**
- * TODO!
+ * Adds debugging information to a request.
*
* @version $Id$
* @since solr 1.3
@@ -42,48 +41,182 @@
public static final String COMPONENT_NAME = "debug";
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void prepare(ResponseBuilder rb) throws IOException
{
}
@SuppressWarnings("unchecked")
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void process(ResponseBuilder rb) throws IOException
{
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
- if( builder.isDebug() ) {
- NamedList stdinfo = SolrPluginUtils.doStandardDebug( req,
- builder.getQueryString(), builder.getQuery(), builder.getResults().docList);
+ if( rb.isDebug() ) {
+ NamedList stdinfo = SolrPluginUtils.doStandardDebug( rb.req,
+ rb.getQueryString(), rb.getQuery(), rb.getResults().docList);
- NamedList info = builder.getDebugInfo();
+ NamedList info = rb.getDebugInfo();
if( info == null ) {
- builder.setDebugInfo( stdinfo );
+ rb.setDebugInfo( stdinfo );
info = stdinfo;
}
else {
info.addAll( stdinfo );
}
- if (builder.getQparser() != null) {
- builder.getQparser().addDebugInfo(builder.getDebugInfo());
+ if (rb.getQparser() != null) {
+ rb.getQparser().addDebugInfo(rb.getDebugInfo());
}
- if (null != builder.getDebugInfo() ) {
- if (null != builder.getFilters() ) {
- info.add("filter_queries",req.getParams().getParams(FQ));
- List<String> fqs = new ArrayList<String>(builder.getFilters().size());
- for (Query fq : builder.getFilters()) {
- fqs.add(QueryParsing.toString(fq, req.getSchema()));
+ if (null != rb.getDebugInfo() ) {
+ if (null != rb.getFilters() ) {
+ info.add("filter_queries",rb.req.getParams().getParams(FQ));
+ List<String> fqs = new ArrayList<String>(rb.getFilters().size());
+ for (Query fq : rb.getFilters()) {
+ fqs.add(QueryParsing.toString(fq, rb.req.getSchema()));
}
info.add("parsed_filter_queries",fqs);
}
// Add this directly here?
- rsp.add("debug", builder.getDebugInfo() );
+ rb.rsp.add("debug", rb.getDebugInfo() );
}
}
}
+
+
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
+ if (!rb.isDebug()) return;
+
+ // Turn on debug to get explain only only when retrieving fields
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+ sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG;
+ sreq.params.set("debugQuery", "true");
+ } else {
+ sreq.params.set("debugQuery", "false");
+ }
+ }
+
+ @Override
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ }
+
+ private Set<String> excludeSet = new HashSet<String>(Arrays.asList("explain"));
+
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.isDebug() && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ NamedList info = null;
+ NamedList explain = new SimpleOrderedMap();
+
+ Object[] arr = new Object[rb.resultIds.size() * 2];
+
+ for (ShardRequest sreq : rb.finished) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_DEBUG) == 0) continue;
+ for (ShardResponse srsp : sreq.responses) {
+ NamedList sdebug = (NamedList)srsp.rsp.getResponse().get("debug");
+ info = (NamedList)merge(sdebug, info, excludeSet);
+
+ NamedList sexplain = (NamedList)sdebug.get("explain");
+
+ for (int i=0; i<sexplain.size(); i++) {
+ String id = sexplain.getName(i);
+ // TODO: lookup won't work for non-string ids... String vs Float
+ ShardDoc sdoc = rb.resultIds.get(id);
+ int idx = sdoc.positionInResponse;
+ arr[idx<<1] = id;
+ arr[(idx<<1)+1] = sexplain.getVal(i);
+ }
+ }
+ }
+
+ explain = HighlightComponent.removeNulls(new NamedList(Arrays.asList(arr)));
+ int idx = info.indexOf("explain",0);
+ if (idx>=0) {
+ info.setVal(idx, explain);
+ } else {
+ info.add("explain", explain);
+ }
+
+ rb.setDebugInfo(info);
+ rb.rsp.add("debug", rb.getDebugInfo() );
+ }
+ }
+
+
+ Object merge(Object source, Object dest, Set<String> exclude) {
+ if (source == null) return dest;
+ if (dest == null) {
+ if (source instanceof NamedList) {
+ dest = source instanceof SimpleOrderedMap ? new SimpleOrderedMap() : new NamedList();
+ } else {
+ return source;
+ }
+ } else {
+
+ if (dest instanceof Collection) {
+ if (source instanceof Collection) {
+ ((Collection)dest).addAll((Collection)source);
+ } else {
+ ((Collection)dest).add(source);
+ }
+ return dest;
+ } else if (source instanceof Number) {
+ if (dest instanceof Number) {
+ if (source instanceof Double || dest instanceof Double) {
+ return ((Number)source).doubleValue() + ((Number)dest).doubleValue();
+ }
+ return ((Number)source).longValue() + ((Number)dest).longValue();
+ }
+ // fall through
+ } else if (source instanceof String) {
+ if (source.equals(dest)) {
+ return dest;
+ }
+ // fall through
+ }
+ }
+
+
+ if (source instanceof NamedList && dest instanceof NamedList) {
+ NamedList tmp = new NamedList();
+ NamedList sl = (NamedList)source;
+ NamedList dl = (NamedList)dest;
+ for (int i=0; i<sl.size(); i++) {
+ String skey = sl.getName(i);
+ if (exclude != null && exclude.contains(skey)) continue;
+ Object sval = sl.getVal(i);
+ int didx = -1;
+
+ // optimize case where elements are in same position
+ if (i < dl.size()) {
+ String dkey = dl.getName(i);
+ if (skey == dkey || (skey!=null && skey.equals(dkey))) {
+ didx = i;
+ }
+ }
+
+ if (didx == -1) {
+ didx = dl.indexOf(skey, 0);
+ }
+
+ if (didx == -1) {
+ tmp.add(skey, merge(sval, null, null));
+ } else {
+ dl.setVal(didx, merge(sval, dl.getVal(didx), null));
+ }
+ }
+ dl.addAll(tmp);
+ return dl;
+ }
+
+ // merge unlike elements in a list
+ List t = new ArrayList();
+ t.add(dest);
+ t.add(source);
+ return t;
+ }
+
+
/////////////////////////////////////////////
/// SolrInfoMBean
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/FacetComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/FacetComponent.java Tue Feb 26 11:47:07 2008
@@ -19,12 +19,19 @@
import java.io.IOException;
import java.net.URL;
+import java.util.*;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.SolrException;
import org.apache.solr.request.SimpleFacets;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.util.OpenBitSet;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.QueryParsing;
+import org.apache.lucene.queryParser.ParseException;
/**
* TODO!
@@ -37,34 +44,336 @@
public static final String COMPONENT_NAME = "facet";
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void prepare(ResponseBuilder rb) throws IOException
{
- SolrParams params = req.getParams();
- if (params.getBool(FacetParams.FACET,false)) {
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
- builder.setNeedDocSet( true );
+ if (rb.req.getParams().getBool(FacetParams.FACET,false)) {
+ rb.setNeedDocSet( true );
+ rb.doFacets = true;
}
}
/**
* Actually run the query
+ * @param rb
*/
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void process(ResponseBuilder rb) throws IOException
{
- SolrParams params = req.getParams();
- if (params.getBool(FacetParams.FACET,false)) {
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
-
- SimpleFacets f = new SimpleFacets(req.getSearcher(),
- builder.getResults().docSet,
- params );
+ if (rb.doFacets) {
+ SolrParams params = rb.req.getParams();
+ SimpleFacets f = new SimpleFacets(rb.req,
+ rb.getResults().docSet,
+ params );
+
+ // TODO ???? add this directly to the response, or to the builder?
+ rb.rsp.add( "facet_counts", f.getFacetCounts() );
+ }
+ }
+
+
+ @Override
+ public int distributedProcess(ResponseBuilder rb) throws IOException {
+ if (!rb.doFacets) {
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ // overlap facet refinement requests (those shards that we need a count for
+ // particular facet values from), where possible, with
+ // the requests to get fields (because we know that is the
+ // only other required phase).
+ // We do this in distributedProcess so we can look at all of the
+ // requests in the outgoing queue at once.
+
+ for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
+ List<String> fqueries = rb._facetInfo._toRefine[shardNum];
+ if (fqueries == null || fqueries.size()==0) continue;
+
+ String shard = rb.shards[shardNum];
+
+ ShardRequest refine = null;
+ boolean newRequest = false;
+
+ // try to find a request that is already going out to that shard.
+ // If nshards becomes to great, we way want to move to hashing for better
+ // scalability.
+ for (ShardRequest sreq : rb.outgoing) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS)!=0
+ && sreq.shards != null & sreq.shards.length==1
+ && sreq.shards[0].equals(shard))
+ {
+ refine = sreq;
+ break;
+ }
+ }
+
+ if (refine == null) {
+ // we didn't find any other suitable requests going out to that shard, so
+ // create one ourselves.
+ newRequest = true;
+ refine = new ShardRequest();
+ refine.shards = new String[]{rb.shards[shardNum]};
+ refine.params = new ModifiableSolrParams(rb.req.getParams());
+ // don't request any documents
+ refine.params.remove("start");
+ refine.params.set("rows","0");
+ }
+
+ refine.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
+ refine.params.set(FacetParams.FACET,"true");
+ refine.params.remove(FacetParams.FACET_FIELD);
+ // TODO: perhaps create a more compact facet.terms method?
+ refine.params.set(FacetParams.FACET_QUERY, fqueries.toArray(new String[fqueries.size()]));
+
+ if (newRequest) {
+ rb.addRequest(this, refine);
+ }
+ }
+ }
+
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ @Override
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
+ if (!rb.doFacets) return;
+
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ sreq.purpose |= ShardRequest.PURPOSE_GET_FACETS;
+
+ FacetInfo fi = rb._facetInfo;
+ if (fi == null) {
+ rb._facetInfo = fi = new FacetInfo();
+ fi.parse(rb.req.getParams(), rb);
+ // should already be true...
+ // sreq.params.set(FacetParams.FACET, "true");
+ }
+
+ sreq.params.remove(FacetParams.FACET_MINCOUNT);
+ sreq.params.remove(FacetParams.FACET_OFFSET);
+ sreq.params.remove(FacetParams.FACET_LIMIT);
+
+ for (DistribFieldFacet dff : fi.topFacets.values()) {
+ String paramStart = "f." + dff.field + '.';
+ sreq.params.remove(paramStart + FacetParams.FACET_MINCOUNT);
+ sreq.params.remove(paramStart + FacetParams.FACET_OFFSET);
+
+ // set the initial limit higher in increase accuracy
+ dff.initialLimit = dff.offset + dff.limit;
+ dff.initialLimit = (int)(dff.initialLimit * 1.5) + 10;
+
+ // Uncomment the following line when testing to supress over-requesting facets and
+ // thus cause more facet refinement queries.
+ // dff.initialLimit = dff.offset + dff.limit;
+
+ sreq.params.set(paramStart + FacetParams.FACET_LIMIT, dff.initialLimit);
+ }
+ } else {
+ // turn off faceting on other requests
+ sreq.params.set(FacetParams.FACET, "false");
+ // we could optionally remove faceting params
+ }
+ }
+
+ @Override
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ if (!rb.doFacets) return;
+
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FACETS)!=0) {
+ countFacets(rb, sreq);
+ } else if ((sreq.purpose & ShardRequest.PURPOSE_REFINE_FACETS)!=0) {
+ refineFacets(rb, sreq);
+ }
+ }
+
+
+
+
+ private void countFacets(ResponseBuilder rb, ShardRequest sreq) {
+ FacetInfo fi = rb._facetInfo;
+
+ for (ShardResponse srsp: sreq.responses) {
+ int shardNum = rb.getShardNum(srsp.shard);
+ NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
+
+ // handle facet queries
+ NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
+ if (facet_queries != null) {
+ for (int i=0; i<facet_queries.size(); i++) {
+ String facet_q = (String)facet_queries.getName(i);
+ long count = ((Number)facet_queries.getVal(i)).longValue();
+ Long prevCount = fi.queryFacets.get(facet_q);
+ if (prevCount != null) count += prevCount;
+ fi.queryFacets.put(facet_q, count);
+ }
+ }
+
+ // step through each facet.field, adding results from this shard
+ NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
+ for (DistribFieldFacet dff : fi.topFacets.values()) {
+ dff.add(shardNum, (NamedList)facet_fields.get(dff.field), dff.initialLimit);
+ }
+ }
+
+
+ //
+ // This code currently assumes that there will be only a single
+ // request ((with responses from all shards) sent out to get facets...
+ // otherwise we would need to wait until all facet responses were received.
+ //
+
+ // list of queries to send each shard
+ List<String>[] toRefine = new List[rb.shards.length];
+ fi._toRefine = toRefine;
+ for (int i=0; i<toRefine.length; i++) {
+ toRefine[i] = new ArrayList<String>();
+ }
+
+
+ for (DistribFieldFacet dff : fi.topFacets.values()) {
+ ShardFacetCount[] counts = dff.getSorted();
+ int ntop = Math.min(counts.length, dff.offset + dff.limit);
+ long smallestCount = counts[ntop-1].count;
+
+ for (int i=0; i<counts.length; i++) {
+ ShardFacetCount sfc = counts[i];
+ String query = null;
+ boolean needRefinement = false;
+
+ if (i<ntop) {
+ // automatically flag the top values for refinement
+ needRefinement = true;
+ } else {
+ // calculate the maximum value that this term may have
+ // and if it is >= smallestCount, then flag for refinement
+ long maxCount = sfc.count;
+ for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
+ OpenBitSet obs = dff.counted[shardNum];
+ if (!obs.get(sfc.termNum)) {
+ // if missing from this shard, add the max it could be
+ maxCount += dff.maxPossible(sfc,shardNum);
+ }
+ }
+ if (maxCount >= smallestCount) {
+ // TODO: on a tie, we could check the term values
+ needRefinement = true;
+ }
+ }
+
+ if (needRefinement) {
+ // add a query for each shard missing the term that needs refinement
+ for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
+ OpenBitSet obs = dff.counted[shardNum];
+ if (!obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
+ dff.needRefinements = true;
+ if (query==null) query = dff.makeQuery(sfc);
+ toRefine[shardNum].add(query);
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ private void refineFacets(ResponseBuilder rb, ShardRequest sreq) {
+ FacetInfo fi = rb._facetInfo;
+
+ for (ShardResponse srsp: sreq.responses) {
+ int shardNum = rb.getShardNum(srsp.shard);
+ NamedList facet_counts = (NamedList)srsp.rsp.getResponse().get("facet_counts");
+ NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
+
+ // These are single term queries used to fill in missing counts
+ // for facet.field queries
+ for (int i=0; i<facet_queries.size(); i++) {
+ try {
+
+ String facet_q = (String)facet_queries.getName(i);
+ long count = ((Number)facet_queries.getVal(i)).longValue();
+
+ // expect <!field f=field>value style params
+ SolrParams qparams = QueryParsing.getLocalParams(facet_q,null);
+ String field = qparams.get(QueryParsing.F);
+ String val = qparams.get(QueryParsing.V);
+
+ // Find the right field.facet for this field
+ DistribFieldFacet dff = fi.topFacets.get(field);
+ // Find the right constraint count for this value
+ ShardFacetCount sfc = dff.counts.get(val);
+// TODO REMOVE
+System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + count);
+
+ sfc.count += count;
- // TODO ???? add this directly to the response?
- rsp.add( "facet_counts", f.getFacetCounts() );
+ } catch (ParseException e) {
+ // shouldn't happen, so fail for now rather than covering it up
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
}
}
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ if (!rb.doFacets || rb.stage != ResponseBuilder.STAGE_GET_FIELDS) return;
+ // wait until STAGE_GET_FIELDS
+ // so that "result" is already stored in the response (for aesthetics)
+
+
+ FacetInfo fi = rb._facetInfo;
+
+ NamedList facet_counts = new NamedList();
+ NamedList facet_queries = new NamedList();
+ facet_counts.add("facet_queries",facet_queries);
+ for (Map.Entry<String,Long> entry : fi.queryFacets.entrySet()) {
+ facet_queries.add(entry.getKey(), num(entry.getValue()));
+ }
+
+ NamedList facet_fields = new NamedList();
+ facet_counts.add("facet_fields", facet_fields);
+
+ for (DistribFieldFacet dff : fi.topFacets.values()) {
+ SimpleOrderedMap fieldCounts = new SimpleOrderedMap();
+ facet_fields.add(dff.field, fieldCounts);
+
+ ShardFacetCount[] counts = dff.countSorted;
+ if (dff.needRefinements) {
+ counts = dff.getSorted();
+ }
+
+ int end = Math.min(dff.offset + dff.limit, counts.length);
+ for (int i=dff.offset; i<end; i++) {
+ if (counts[i].count < dff.minCount) break;
+ fieldCounts.add(counts[i].name, num(counts[i].count));
+ }
+
+ if (dff.missing) {
+ fieldCounts.add(null, num(dff.missingCount));
+ }
+ }
+
+ // TODO: list facets (sorted by natural order)
+ // TODO: facet dates
+ facet_counts.add("facet_dates", new NamedList());
+
+ rb.rsp.add("facet_counts", facet_counts);
+
+ rb._facetInfo = null; // could be big, so release asap
+ }
+
+
+ // use <int> tags for smaller facet counts (better back compatibility)
+ private Number num(long val) {
+ if (val < Integer.MAX_VALUE) return (int)val;
+ else return val;
+ }
+ private Number num(Long val) {
+ if (val.longValue() < Integer.MAX_VALUE) return val.intValue();
+ else return val;
+ }
+
+
/////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
@@ -92,5 +401,173 @@
@Override
public URL[] getDocs() {
return null;
+ }
+}
+
+
+
+class FacetInfo {
+ List<String>[] _toRefine;
+
+ void parse(SolrParams params, ResponseBuilder rb) {
+ queryFacets = new LinkedHashMap<String,Long>();
+ topFacets = new LinkedHashMap<String,DistribFieldFacet>();
+ listFacets = new LinkedHashMap<String,DistribFieldFacet>();
+
+ String[] facetQs = params.getParams(FacetParams.FACET_QUERY);
+ if (facetQs != null) {
+ for (String query : facetQs) {
+ queryFacets.put(query,0L);
+ }
+ }
+
+ String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
+ if (facetFs != null) {
+ for (String field : facetFs) {
+ DistribFieldFacet ff = new DistribFieldFacet(rb, field);
+ ff.fillParams(params, field);
+ if (ff.sort) {
+ topFacets.put(field, ff);
+ } else {
+ listFacets.put(field, ff);
+ }
+ }
+ }
+ }
+
+ LinkedHashMap<String,Long> queryFacets;
+ LinkedHashMap<String,DistribFieldFacet> topFacets; // field facets that order by constraint count (sort=true)
+ LinkedHashMap<String,DistribFieldFacet> listFacets; // field facets that list values in term order
+}
+
+
+class FieldFacet {
+ String field;
+ int offset;
+ int limit;
+ int minCount;
+ boolean sort;
+ boolean missing;
+ String prefix;
+ long missingCount;
+
+ void fillParams(SolrParams params, String field) {
+ this.field = field;
+ this.offset = params.getFieldInt(field, FacetParams.FACET_OFFSET, 0);
+ this.limit = params.getFieldInt(field, FacetParams.FACET_LIMIT, 100);
+ Integer mincount = params.getFieldInt(field, FacetParams.FACET_MINCOUNT);
+ if (mincount==null) {
+ Boolean zeros = params.getFieldBool(field, FacetParams.FACET_ZEROS);
+ // mincount = (zeros!=null && zeros) ? 0 : 1;
+ mincount = (zeros!=null && !zeros) ? 1 : 0;
+ // current default is to include zeros.
+ }
+ this.minCount = mincount;
+ this.missing = params.getFieldBool(field, FacetParams.FACET_MISSING, false);
+ // default to sorting if there is a limit.
+ this.sort = params.getFieldBool(field, FacetParams.FACET_SORT, limit>0);
+ this.prefix = params.getFieldParam(field,FacetParams.FACET_PREFIX);
+ }
+}
+
+class DistribFieldFacet extends FieldFacet {
+ SchemaField sf;
+
+ // the max possible count for a term appearing on no list
+ long missingMaxPossible;
+ // the max possible count for a missing term for each shard (indexed by shardNum)
+ long[] missingMax;
+ OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
+ HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
+ int termNum;
+ String queryPrefix;
+
+ int initialLimit; // how many terms requested in first phase
+ boolean needRefinements;
+ ShardFacetCount[] countSorted;
+
+ DistribFieldFacet(ResponseBuilder rb, String field) {
+ sf = rb.req.getSchema().getField(field);
+ missingMax = new long[rb.shards.length];
+ counted = new OpenBitSet[rb.shards.length];
+ queryPrefix = "<!field f=" + field + '>';
+ }
+
+ void add(int shardNum, NamedList shardCounts, int numRequested) {
+ int sz = shardCounts.size();
+ int numReceived = sz;
+
+ OpenBitSet terms = new OpenBitSet(termNum+sz);
+
+ long last = 0;
+ for (int i=0; i<sz; i++) {
+ String name = shardCounts.getName(i);
+ long count = ((Number)shardCounts.getVal(i)).longValue();
+ if (name == null) {
+ missingCount += count;
+ numReceived--;
+ } else {
+ ShardFacetCount sfc = counts.get(name);
+ if (sfc == null) {
+ sfc = new ShardFacetCount();
+ sfc.name = name;
+ sfc.termNum = termNum++;
+ counts.put(name, sfc);
+ }
+ sfc.count += count;
+ terms.fastSet(sfc.termNum);
+ last = count;
+ }
+ }
+
+ // the largest possible missing term is 0 if we received less
+ // than the number requested (provided mincount==0 like it should be for
+ // a shard request)
+ if (numRequested !=0 && numReceived < numRequested) {
+ last = 0;
+ }
+
+ missingMaxPossible += last;
+ missingMax[shardNum] = last;
+ counted[shardNum] = terms;
+ }
+
+
+ ShardFacetCount[] getSorted() {
+ ShardFacetCount[] arr = counts.values().toArray(new ShardFacetCount[counts.size()]);
+ Arrays.sort(arr, new Comparator<ShardFacetCount>() {
+ public int compare(ShardFacetCount o1, ShardFacetCount o2) {
+ if (o2.count < o1.count) return -1;
+ else if (o1.count < o2.count) return 1;
+ // TODO: handle tiebreaks for types other than strings
+ return o1.name.compareTo(o2.name);
+ }
+ });
+ countSorted = arr;
+ return arr;
+ }
+
+ String makeQuery(ShardFacetCount sfc) {
+ return queryPrefix + sfc.name;
+ }
+
+ // returns the max possible value this ShardFacetCount could have for this shard
+ // (assumes the shard did not report a count for this value)
+ long maxPossible(ShardFacetCount sfc, int shardNum) {
+ return missingMax[shardNum];
+ // TODO: could store the last term in the shard to tell if this term
+ // comes before or after it. If it comes before, we could subtract 1
+ }
+
+}
+
+
+class ShardFacetCount {
+ String name;
+ long count;
+ int termNum; // term number starting at 0 (used in bit arrays)
+
+ public String toString() {
+ return "{term="+name+",termNum="+termNum+",count="+count+"}";
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/HighlightComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/HighlightComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/HighlightComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/HighlightComponent.java Tue Feb 26 11:47:07 2008
@@ -19,14 +19,16 @@
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.highlight.SolrHighlighter;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrQueryResponse;
import java.io.IOException;
import java.net.URL;
+import java.util.Arrays;
/**
* TODO!
@@ -39,51 +41,113 @@
public static final String COMPONENT_NAME = "highlight";
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp)
+ public void prepare(ResponseBuilder rb) throws IOException
{
-
+ SolrHighlighter highlighter = rb.req.getCore().getHighlighter();
+ rb.doHighlights = highlighter.isHighlightingEnabled(rb.req.getParams());
}
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
- SolrHighlighter highlighter = req.getCore().getHighlighter();
- if (highlighter.isHighlightingEnabled(req.getParams())) {
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
+ public void process(ResponseBuilder rb) throws IOException {
+ SolrQueryRequest req = rb.req;
+ if (rb.doHighlights) {
+ SolrHighlighter highlighter = req.getCore().getHighlighter();
SolrParams params = req.getParams();
String[] defaultHighlightFields; //TODO: get from builder by default?
- if (builder.getQparser() != null) {
- defaultHighlightFields = builder.getQparser().getDefaultHighlightFields();
+ if (rb.getQparser() != null) {
+ defaultHighlightFields = rb.getQparser().getDefaultHighlightFields();
} else {
defaultHighlightFields = params.getParams(CommonParams.DF);
}
- if(builder.getHighlightQuery()==null) {
- if (builder.getQparser() != null) {
+ if(rb.getHighlightQuery()==null) {
+ if (rb.getQparser() != null) {
try {
- builder.setHighlightQuery( builder.getQparser().getHighlightQuery() );
+ rb.setHighlightQuery( rb.getQparser().getHighlightQuery() );
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
} else {
- builder.setHighlightQuery( builder.getQuery() );
+ rb.setHighlightQuery( rb.getQuery() );
}
}
NamedList sumData = highlighter.doHighlighting(
- builder.getResults().docList,
- builder.getHighlightQuery().rewrite(req.getSearcher().getReader()),
+ rb.getResults().docList,
+ rb.getHighlightQuery().rewrite(req.getSearcher().getReader()),
req, defaultHighlightFields );
if(sumData != null) {
// TODO ???? add this directly to the response?
- rsp.add("highlighting", sumData);
+ rb.rsp.add("highlighting", sumData);
}
}
}
-
- /////////////////////////////////////////////
+
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
+ if (!rb.doHighlights) return;
+
+ // Turn on highlighting only only when retrieving fields
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+ sreq.purpose |= ShardRequest.PURPOSE_GET_HIGHLIGHTS;
+ // should already be true...
+ sreq.params.set(HighlightParams.HIGHLIGHT, "true");
+ } else {
+ sreq.params.set(HighlightParams.HIGHLIGHT, "false");
+ }
+ }
+
+ @Override
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ }
+
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ NamedList hlResult = new SimpleOrderedMap();
+
+ Object[] arr = new Object[rb.resultIds.size() * 2];
+
+ // TODO: make a generic routine to do automatic merging of id keyed data
+ for (ShardRequest sreq : rb.finished) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
+ for (ShardResponse srsp : sreq.responses) {
+ NamedList hl = (NamedList)srsp.rsp.getResponse().get("highlighting");
+ for (int i=0; i<hl.size(); i++) {
+ String id = hl.getName(i);
+ ShardDoc sdoc = rb.resultIds.get(id);
+ int idx = sdoc.positionInResponse;
+ arr[idx<<1] = id;
+ arr[(idx<<1)+1] = hl.getVal(i);
+ }
+ }
+ }
+
+ // remove nulls in case not all docs were able to be retrieved
+ rb.rsp.add("highlighting", removeNulls(new NamedList(Arrays.asList(arr))));
+ }
+ }
+
+
+ static NamedList removeNulls(NamedList nl) {
+ for (int i=0; i<nl.size(); i++) {
+ if (nl.getName(i)==null) {
+ NamedList newList = new NamedList();
+ for (int j=0; j<nl.size(); j++) {
+ String n = nl.getName(j);
+ if (n != null) {
+ newList.add(n, nl.getVal(j));
+ }
+ }
+ return newList;
+ }
+ }
+ return nl;
+ }
+
+ ////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java Tue Feb 26 11:47:07 2008
@@ -24,8 +24,6 @@
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.MoreLikeThisHandler;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
@@ -40,28 +38,27 @@
public static final String COMPONENT_NAME = "mlt";
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void prepare(ResponseBuilder rb) throws IOException
{
}
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void process(ResponseBuilder rb) throws IOException
{
- SolrParams p = req.getParams();
+ SolrParams p = rb.req.getParams();
if( p.getBool( MoreLikeThisParams.MLT, false ) ) {
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
- SolrIndexSearcher searcher = req.getSearcher();
+ SolrIndexSearcher searcher = rb.req.getSearcher();
MoreLikeThisHandler.MoreLikeThisHelper mlt
= new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher );
int mltcount = p.getInt( MoreLikeThisParams.DOC_COUNT, 5 );
NamedList<DocList> sim = mlt.getMoreLikeThese(
- builder.getResults().docList, mltcount, builder.getFieldFlags() );
+ rb.getResults().docList, mltcount, rb.getFieldFlags() );
// TODO ???? add this directly to the response?
- rsp.add( "moreLikeThis", sim );
+ rb.rsp.add( "moreLikeThis", sim );
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java Tue Feb 26 11:47:07 2008
@@ -17,19 +17,32 @@
package org.apache.solr.handler.component;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.*;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.FieldType;
import org.apache.solr.search.*;
import org.apache.solr.util.SolrPluginUtils;
import java.io.IOException;
+import java.io.Reader;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+import java.text.Collator;
/**
* TODO!
@@ -42,9 +55,10 @@
public static final String COMPONENT_NAME = "query";
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, ParseException
+ public void prepare(ResponseBuilder rb) throws IOException
{
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
// Set field flags
@@ -53,32 +67,43 @@
if (fl != null) {
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
}
- builder.setFieldFlags( fieldFlags );
+ rb.setFieldFlags( fieldFlags );
String defType = params.get(QueryParsing.DEFTYPE);
defType = defType==null ? OldLuceneQParserPlugin.NAME : defType;
- if (builder.getQueryString() == null) {
- builder.setQueryString( params.get( CommonParams.Q ) );
+ if (rb.getQueryString() == null) {
+ rb.setQueryString( params.get( CommonParams.Q ) );
}
- QParser parser = QParser.getParser(builder.getQueryString(), defType, req);
- builder.setQuery( parser.getQuery() );
- builder.setSortSpec( parser.getSort(true) );
-
- String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
- if (fqs!=null && fqs.length!=0) {
- List<Query> filters = builder.getFilters();
- if (filters==null) {
- filters = new ArrayList<Query>();
- builder.setFilters( filters );
- }
- for (String fq : fqs) {
- if (fq != null && fq.trim().length()!=0) {
- QParser fqp = QParser.getParser(fq, null, req);
- filters.add(fqp.getQuery());
+ try {
+ QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
+ rb.setQuery( parser.getQuery() );
+ rb.setSortSpec( parser.getSort(true) );
+
+ String[] fqs = req.getParams().getParams(org.apache.solr.common.params.CommonParams.FQ);
+ if (fqs!=null && fqs.length!=0) {
+ List<Query> filters = rb.getFilters();
+ if (filters==null) {
+ filters = new ArrayList<Query>();
+ rb.setFilters( filters );
+ }
+ for (String fq : fqs) {
+ if (fq != null && fq.trim().length()!=0) {
+ QParser fqp = QParser.getParser(fq, null, req);
+ filters.add(fqp.getQuery());
+ }
}
}
+ } catch (ParseException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+
+ // TODO: temporary... this should go in a different component.
+ String shards = params.get("shards");
+ if (shards != null) {
+ List<String> lst = StrUtils.splitSmart(shards, ",", true);
+ rb.shards = lst.toArray(new String[lst.size()]);
}
}
@@ -86,35 +111,692 @@
* Actually run the query
*/
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void process(ResponseBuilder rb) throws IOException
{
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();
SolrParams params = req.getParams();
-
- if( builder.isNeedDocSet() ) {
- builder.setResults( searcher.getDocListAndSet(
- builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
- builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
- builder.getFieldFlags() ) );
+
+ // Optional: This could also be implemented by the top-level searcher sending
+ // a filter that lists the ids... that would be transparent to
+ // the request handler, but would be more expensive (and would preserve score
+ // too if desired).
+ String ids = params.get("ids");
+ if (ids != null) {
+ SchemaField idField = req.getSchema().getUniqueKeyField();
+ List<String> idArr = StrUtils.splitSmart(ids, ",", true);
+ int[] luceneIds = new int[idArr.size()];
+ int docs = 0;
+ for (int i=0; i<idArr.size(); i++) {
+ int id = req.getSearcher().getFirstMatch(
+ new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
+ if (id >= 0)
+ luceneIds[docs++] = id;
+ }
+
+ DocListAndSet res = new DocListAndSet();
+ res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
+ if (rb.isNeedDocSet()) {
+ List<Query> queries = new ArrayList<Query>();
+ queries.add(rb.getQuery());
+ List<Query> filters = rb.getFilters();
+ if (filters != null) queries.addAll(filters);
+ res.docSet = searcher.getDocSet(queries);
+ }
+ rb.setResults(res);
+ rsp.add("response",rb.getResults().docList);
+ return;
+ }
+
+ if( rb.isNeedDocSet() ) {
+ rb.setResults( searcher.getDocListAndSet(
+ rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
+ rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
+ rb.getFieldFlags() ) );
}
else {
DocListAndSet results = new DocListAndSet();
results.docList = searcher.getDocList(
- builder.getQuery(), builder.getFilters(), builder.getSortSpec().getSort(),
- builder.getSortSpec().getOffset(), builder.getSortSpec().getCount(),
- builder.getFieldFlags() );
- builder.setResults( results );
+ rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
+ rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
+ rb.getFieldFlags() );
+ rb.setResults( results );
+ }
+
+ rsp.add("response",rb.getResults().docList);
+
+
+ boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
+ if(fsv){
+ Sort sort = rb.getSortSpec().getSort();
+ SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
+ ScoreDoc sd = new ScoreDoc(0,1.0f); // won't work for comparators that look at the score
+ NamedList sortVals = new NamedList();
+ StringFieldable field = new StringFieldable();
+
+ for (SortField sortField: sortFields) {
+ int type = sortField.getType();
+ if (type==SortField.SCORE || type==SortField.DOC) continue;
+
+ ScoreDocComparator comparator = null;
+ IndexReader reader = searcher.getReader();
+ String fieldname = sortField.getField();
+ FieldType ft = fieldname==null ? null : req.getSchema().getFieldTypeNoEx(fieldname);
+
+
+ switch (type) {
+ case SortField.INT:
+ comparator = comparatorInt (reader, fieldname);
+ break;
+ case SortField.FLOAT:
+ comparator = comparatorFloat (reader, fieldname);
+ break;
+ case SortField.LONG:
+ comparator = comparatorLong(reader, fieldname);
+ break;
+ case SortField.DOUBLE:
+ comparator = comparatorDouble(reader, fieldname);
+ break;
+ case SortField.STRING:
+ if (sortField.getLocale() != null) comparator = comparatorStringLocale (reader, fieldname, sortField.getLocale());
+ else comparator = comparatorString (reader, fieldname);
+ break;
+ case SortField.CUSTOM:
+ comparator = sortField.getFactory().newComparator (reader, fieldname);
+ break;
+ default:
+ throw new RuntimeException ("unknown field type: "+type);
+ }
+
+ DocList docList = rb.getResults().docList;
+ ArrayList<Object> vals = new ArrayList<Object>(docList.size());
+ DocIterator it = rb.getResults().docList.iterator();
+ while(it.hasNext()) {
+ sd.doc = it.nextDoc();
+ Object val = comparator.sortValue(sd);
+ // Sortable float, double, int, long types all just use a string
+ // comparator. For these, we need to put the type into a readable
+ // format. One reason for this is that XML can't represent all
+ // string values (or even all unicode code points).
+ // indexedToReadable() should be a no-op and should
+ // thus be harmless anyway (for all current ways anyway)
+ if (val instanceof String) {
+ field.val = (String)val;
+ val = ft.toObject(field);
+ }
+ vals.add(val);
+ }
+
+ sortVals.add(fieldname, vals);
+ }
+
+ rsp.add("sort_values", sortVals);
}
//pre-fetch returned documents
- if (builder.getResults().docList != null && builder.getResults().docList.size()<=50) {
+ if (!req.getParams().getBool("isShard",false) && rb.getResults().docList != null && rb.getResults().docList.size()<=50) {
// TODO: this may depend on the highlighter component (or other components?)
- SolrPluginUtils.optimizePreFetchDocs(builder.getResults().docList, builder.getQuery(), req, rsp);
+ SolrPluginUtils.optimizePreFetchDocs(rb.getResults().docList, rb.getQuery(), req, rsp);
}
- rsp.add("response",builder.getResults().docList);
}
-
+
+ @Override
+ public int distributedProcess(ResponseBuilder rb) throws IOException {
+ if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
+ return ResponseBuilder.STAGE_PARSE_QUERY;
+ if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
+ createDistributedIdf(rb);
+ return ResponseBuilder.STAGE_EXECUTE_QUERY;
+ }
+ if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) return ResponseBuilder.STAGE_EXECUTE_QUERY;
+ if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
+ createMainQuery(rb);
+ return ResponseBuilder.STAGE_GET_FIELDS;
+ }
+ if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) return ResponseBuilder.STAGE_GET_FIELDS;
+ if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ createRetrieveDocs(rb);
+ return ResponseBuilder.STAGE_DONE;
+ }
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+
+ @Override
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ mergeIds(rb, sreq);
+ return;
+ }
+
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+ returnFields(rb, sreq);
+ return;
+ }
+ }
+
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ // We may not have been able to retrieve all the docs due to an
+ // index change. Remove any null documents.
+ for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
+ if (iter.next() == null) {
+ iter.remove();
+ rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
+ }
+ }
+
+ rb.rsp.add("response", rb._responseDocs);
+ }
+ }
+
+
+ private void createDistributedIdf(ResponseBuilder rb) {
+ // TODO
+ }
+
+ private void createMainQuery(ResponseBuilder rb) {
+ ShardRequest sreq = new ShardRequest();
+ sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
+
+ sreq.params = new ModifiableSolrParams(rb.req.getParams());
+ // TODO: base on current params or original params?
+
+ // don't pass through any shards param
+ sreq.params.remove("shards");
+
+ // set the start (offset) to 0 for each shard request so we can properly merge
+ // results from the start.
+ sreq.params.set("start","0");
+
+ // TODO: should we even use the SortSpec? That's obtained from the QParser, and
+ // perhaps we shouldn't attempt to parse the query at this level?
+ // Alternate Idea: instead of specifying all these things at the upper level,
+ // we could just specify that this is a shard request.
+ sreq.params.set("rows", rb.getSortSpec().getOffset() + rb.getSortSpec().getCount());
+
+
+ // in this first phase, request only the unique key field
+ // and any fields needed for merging.
+ sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
+
+ if (rb.getSortSpec().includesScore()) {
+ sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName() + ",score");
+ } else {
+ sreq.params.set("fl", rb.req.getSchema().getUniqueKeyField().getName());
+ }
+
+ rb.addRequest(this, sreq);
+ }
+
+
+
+
+
+ private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
+ SortSpec ss = rb.getSortSpec();
+ Sort sort = ss.getSort();
+
+ SortField[] sortFields = null;
+ if(sort != null) sortFields = sort.getSort();
+ else {
+ sortFields = new SortField[]{SortField.FIELD_SCORE};
+ }
+
+ SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
+
+
+ // id to shard mapping, to eliminate any accidental dups
+ HashMap<Object,String> uniqueDoc = new HashMap<Object,String>();
+
+ // Merge the docs via a priority queue so we don't have to sort *all* of the
+ // documents... we only need to order the top (rows+start)
+ ShardFieldSortedHitQueue queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
+
+ long numFound = 0;
+ Float maxScore=null;
+ for (ShardResponse srsp : sreq.responses) {
+ SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
+
+ // calculate global maxScore and numDocsFound
+ if (docs.getMaxScore() != null) {
+ maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
+ }
+ numFound += docs.getNumFound();
+
+ NamedList sortFieldValues = (NamedList)(srsp.rsp.getResponse().get("sort_values"));
+
+ // go through every doc in this response, construct a ShardDoc, and
+ // put it in the priority queue so it can be ordered.
+ for (int i=0; i<docs.size(); i++) {
+ SolrDocument doc = docs.get(i);
+ Object id = doc.getFieldValue(uniqueKeyField.getName());
+
+ String prevShard = uniqueDoc.put(id, srsp.shard);
+ if (prevShard != null) {
+ // duplicate detected
+ numFound--;
+
+ // For now, just always use the first encountered since we can't currently
+ // remove the previous one added to the priority queue. If we switched
+ // to the Java5 PriorityQueue, this would be easier.
+ continue;
+ // make which duplicate is used deterministic based on shard
+ // if (prevShard.compareTo(srsp.shard) >= 0) {
+ // TODO: remove previous from priority queue
+ // continue;
+ // }
+ }
+
+ ShardDoc shardDoc = new ShardDoc();
+ shardDoc.id = id;
+ shardDoc.shard = srsp.shard;
+ shardDoc.orderInShard = i;
+ Object scoreObj = doc.getFieldValue("score");
+ if (scoreObj != null) {
+ if (scoreObj instanceof String) {
+ shardDoc.score = Float.parseFloat((String)scoreObj);
+ } else {
+ shardDoc.score = (Float)scoreObj;
+ }
+ }
+
+ shardDoc.sortFieldValues = sortFieldValues;
+
+ queue.insert(shardDoc);
+ } // end for-each-doc-in-response
+ } // end for-each-response
+
+
+ // The queue now has 0 -> queuesize docs, where queuesize <= start + rows
+ // So we want to pop the last documents off the queue to get
+ // the docs offset -> queuesize
+ int resultSize = queue.size() - ss.getOffset();
+ resultSize = Math.max(0, resultSize); // there may not be any docs in range
+
+ Map<Object,ShardDoc> resultIds = new HashMap<Object,ShardDoc>();
+ for (int i=resultSize-1; i>=0; i--) {
+ ShardDoc shardDoc = (ShardDoc)queue.pop();
+ shardDoc.positionInResponse = i;
+ // Need the toString() for correlation with other lists that must
+ // be strings (like keys in highlighting, explain, etc)
+ resultIds.put(shardDoc.id.toString(), shardDoc);
+ }
+
+
+ SolrDocumentList responseDocs = new SolrDocumentList();
+ if (maxScore!=null) responseDocs.setMaxScore(maxScore);
+ responseDocs.setNumFound(numFound);
+ responseDocs.setStart(ss.getOffset());
+
+ // size appropriately
+ for (int i=0; i<resultSize; i++) responseDocs.add(null);
+
+ // save these results in a private area so we can access them
+ // again when retrieving stored fields.
+ // TODO: use ResponseBuilder (w/ comments) or the request context?
+ rb.resultIds = resultIds;
+ rb._responseDocs = responseDocs;
+ }
+
+ private void createRetrieveDocs(ResponseBuilder rb) {
+
+ // TODO: in a system with nTiers > 2, we could be passed "ids" here
+ // unless those requests always go to the final destination shard
+
+ // for each shard, collect the documents for that shard.
+ HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String,Collection<ShardDoc>>();
+ for (ShardDoc sdoc : rb.resultIds.values()) {
+ Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
+ if (shardDocs == null) {
+ shardDocs = new ArrayList<ShardDoc>();
+ shardMap.put(sdoc.shard, shardDocs);
+ }
+ shardDocs.add(sdoc);
+ }
+
+ SchemaField uniqueField = rb.req.getSchema().getUniqueKeyField();
+
+ // Now create a request for each shard to retrieve the stored fields
+ for (Collection<ShardDoc> shardDocs : shardMap.values()) {
+ ShardRequest sreq = new ShardRequest();
+ sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
+
+ sreq.shards = new String[] {shardDocs.iterator().next().shard};
+
+ sreq.params = new ModifiableSolrParams();
+
+ // add original params
+ sreq.params.add( rb.req.getParams());
+
+ // no need for a sort, we already have order
+ sreq.params.remove("sort");
+
+ // we already have the field sort values
+ sreq.params.remove(ResponseBuilder.FIELD_SORT_VALUES);
+
+ // make sure that the id is returned for correlation
+ String fl = sreq.params.get("fl");
+ if (fl != null) {
+ sreq.params.set("fl", fl+','+uniqueField.getName());
+ }
+
+ ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
+ for (ShardDoc shardDoc : shardDocs) {
+ // TODO: depending on the type, we may need more tha a simple toString()?
+ ids.add(shardDoc.id.toString());
+ }
+ sreq.params.add("ids", StrUtils.join(ids, ','));
+
+ rb.addRequest(this, sreq);
+ }
+
+ }
+
+
+ private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
+ // Keep in mind that this could also be a shard in a multi-tiered system.
+ // TODO: if a multi-tiered system, it seems like some requests
+ // could/should bypass middlemen (like retrieving stored fields)
+ // TODO: merge fsv to if requested
+
+
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+ boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+
+ assert(sreq.responses.size() == 1);
+ ShardResponse srsp = sreq.responses.get(0);
+ SolrDocumentList docs = (SolrDocumentList)srsp.rsp.getResponse().get("response");
+
+ String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName();
+
+ for (SolrDocument doc : docs) {
+ Object id = doc.getFieldValue(keyFieldName);
+ ShardDoc sdoc = rb.resultIds.get(id.toString());
+ if (returnScores && sdoc.score != null) {
+ doc.setField("score", sdoc.score);
+ }
+ rb._responseDocs.set(sdoc.positionInResponse, doc);
+ }
+ }
+ }
+
+
+
+ /////////////////////////////////////////////
+ /// Comparators copied from Lucene
+ /////////////////////////////////////////////
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing integers.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg integer values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorInt (final IndexReader reader, final String fieldname)
+ throws IOException {
+ final String field = fieldname.intern();
+ final int[] fieldOrder = FieldCache.DEFAULT.getInts (reader, field);
+ return new ScoreDocComparator() {
+
+ public final int compare (final ScoreDoc i, final ScoreDoc j) {
+ final int fi = fieldOrder[i.doc];
+ final int fj = fieldOrder[j.doc];
+ if (fi < fj) return -1;
+ if (fi > fj) return 1;
+ return 0;
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return new Integer (fieldOrder[i.doc]);
+ }
+
+ public int sortType() {
+ return SortField.INT;
+ }
+ };
+ }
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing integers.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg integer values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorLong (final IndexReader reader, final String fieldname)
+ throws IOException {
+ final String field = fieldname.intern();
+ final long[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getLongs (reader, field);
+ return new ScoreDocComparator() {
+
+ public final int compare (final ScoreDoc i, final ScoreDoc j) {
+ final long li = fieldOrder[i.doc];
+ final long lj = fieldOrder[j.doc];
+ if (li < lj) return -1;
+ if (li > lj) return 1;
+ return 0;
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return new Long(fieldOrder[i.doc]);
+ }
+
+ public int sortType() {
+ return SortField.LONG;
+ }
+ };
+ }
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing floats.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg float values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorFloat (final IndexReader reader, final String fieldname)
+ throws IOException {
+ final String field = fieldname.intern();
+ final float[] fieldOrder = FieldCache.DEFAULT.getFloats (reader, field);
+ return new ScoreDocComparator () {
+
+ public final int compare (final ScoreDoc i, final ScoreDoc j) {
+ final float fi = fieldOrder[i.doc];
+ final float fj = fieldOrder[j.doc];
+ if (fi < fj) return -1;
+ if (fi > fj) return 1;
+ return 0;
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return new Float (fieldOrder[i.doc]);
+ }
+
+ public int sortType() {
+ return SortField.FLOAT;
+ }
+ };
+ }
+
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing doubles.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg float values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorDouble(final IndexReader reader, final String fieldname)
+ throws IOException {
+ final String field = fieldname.intern();
+ final double[] fieldOrder = ExtendedFieldCache.EXT_DEFAULT.getDoubles (reader, field);
+ return new ScoreDocComparator () {
+
+ public final int compare (final ScoreDoc i, final ScoreDoc j) {
+ final double di = fieldOrder[i.doc];
+ final double dj = fieldOrder[j.doc];
+ if (di < dj) return -1;
+ if (di > dj) return 1;
+ return 0;
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return new Double (fieldOrder[i.doc]);
+ }
+
+ public int sortType() {
+ return SortField.DOUBLE;
+ }
+ };
+ }
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing strings.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg string values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorString (final IndexReader reader, final String fieldname)
+ throws IOException {
+ final String field = fieldname.intern();
+ final FieldCache.StringIndex index = FieldCache.DEFAULT.getStringIndex (reader, field);
+ return new ScoreDocComparator () {
+
+ public final int compare (final ScoreDoc i, final ScoreDoc j) {
+ final int fi = index.order[i.doc];
+ final int fj = index.order[j.doc];
+ if (fi < fj) return -1;
+ if (fi > fj) return 1;
+ return 0;
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return index.lookup[index.order[i.doc]];
+ }
+
+ public int sortType() {
+ return SortField.STRING;
+ }
+ };
+ }
+
+ /**
+ * Returns a comparator for sorting hits according to a field containing strings.
+ * @param reader Index to use.
+ * @param fieldname Fieldable containg string values.
+ * @return Comparator for sorting hits.
+ * @throws IOException If an error occurs reading the index.
+ */
+ static ScoreDocComparator comparatorStringLocale (final IndexReader reader, final String fieldname, final Locale locale)
+ throws IOException {
+ final Collator collator = Collator.getInstance (locale);
+ final String field = fieldname.intern();
+ final String[] index = FieldCache.DEFAULT.getStrings (reader, field);
+ return new ScoreDocComparator() {
+
+ public final int compare(final ScoreDoc i, final ScoreDoc j) {
+ String is = index[i.doc];
+ String js = index[j.doc];
+ if (is == js) {
+ return 0;
+ } else if (is == null) {
+ return -1;
+ } else if (js == null) {
+ return 1;
+ } else {
+ return collator.compare(is, js);
+ }
+ }
+
+ public Comparable sortValue (final ScoreDoc i) {
+ return index[i.doc];
+ }
+
+ public int sortType() {
+ return SortField.STRING;
+ }
+ };
+ }
+
+ static class StringFieldable implements Fieldable {
+ public String val;
+
+ public void setBoost(float boost) {
+ }
+
+ public float getBoost() {
+ return 0;
+ }
+
+ public String name() {
+ return null;
+ }
+
+ public String stringValue() {
+ return val;
+ }
+
+ public Reader readerValue() {
+ return null;
+ }
+
+ public byte[] binaryValue() {
+ return new byte[0];
+ }
+
+ public TokenStream tokenStreamValue() {
+ return null;
+ }
+
+ public boolean isStored() {
+ return true;
+ }
+
+ public boolean isIndexed() {
+ return true;
+ }
+
+ public boolean isTokenized() {
+ return true;
+ }
+
+ public boolean isCompressed() {
+ return false;
+ }
+
+ public boolean isTermVectorStored() {
+ return false;
+ }
+
+ public boolean isStoreOffsetWithTermVector() {
+ return false;
+ }
+
+ public boolean isStorePositionWithTermVector() {
+ return false;
+ }
+
+ public boolean isBinary() {
+ return false;
+ }
+
+ public boolean getOmitNorms() {
+ return false;
+ }
+
+ public void setOmitNorms(boolean omitNorms) {
+ }
+
+ public boolean isLazy() {
+ return false;
+ }
+ }
+
/////////////////////////////////////////////
/// SolrInfoMBean
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Tue Feb 26 11:47:07 2008
@@ -25,7 +25,6 @@
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -59,13 +58,12 @@
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SortSpec;
import org.apache.solr.util.VersionedFile;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.solr.request.SolrQueryRequest;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -304,22 +302,22 @@
//---------------------------------------------------------------------------------
@Override
- public void prepare(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
+ public void prepare(ResponseBuilder rb) throws IOException
{
+ SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
// A runtime param can skip
if( !params.getBool( ENABLE, true ) ) {
return;
}
- ResponseBuilder builder = SearchHandler.getResponseBuilder( req );
- Query query = builder.getQuery();
+ Query query = rb.getQuery();
if( query == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"The QueryElevationComponent needs to be registered 'after' the query component" );
}
- String qstr = getAnalyzedQuery( builder.getQueryString() );
+ String qstr = getAnalyzedQuery( rb.getQueryString() );
IndexReader reader = req.getSearcher().getReader();
ElevationObj booster = null;
try {
@@ -340,11 +338,11 @@
newq.add( bq );
}
}
- builder.setQuery( newq );
+ rb.setQuery( newq );
// if the sort is 'score desc' use a custom sorting method to
// insert documents in their proper place
- SortSpec sortSpec = builder.getSortSpec();
+ SortSpec sortSpec = rb.getSortSpec();
if( sortSpec.getSort() == null ) {
sortSpec.setSort( new Sort( new SortField[] {
new SortField(idField, new ElevationComparatorSource(booster.priority), false ),
@@ -377,7 +375,7 @@
}
// Add debugging information
- if( builder.isDebug() ) {
+ if( rb.isDebug() ) {
List<String> match = null;
if( booster != null ) {
// Extract the elevated terms into a list
@@ -391,12 +389,12 @@
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
dbg.add( "q", qstr );
dbg.add( "match", match );
- builder.addDebugInfo( "queryBoosting", dbg );
+ rb.addDebugInfo( "queryBoosting", dbg );
}
}
@Override
- public void process(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ public void process(ResponseBuilder rb) throws IOException {
// Do nothing -- the real work is modifying the input query
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java Tue Feb 26 11:47:07 2008
@@ -18,23 +18,34 @@
package org.apache.solr.handler.component;
import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RTimer;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SortSpec;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
- * TODO!
+ * This class is experimental and will be changing in the future.
*
* @version $Id$
* @since solr 1.3
*/
public class ResponseBuilder
{
+ public SolrQueryRequest req;
+ public SolrQueryResponse rsp;
+ public boolean doHighlights;
+ public boolean doFacets;
+
private boolean needDocList = false;
private boolean needDocSet = false;
private int fieldFlags = 0;
@@ -51,11 +62,74 @@
private RTimer timer = null;
private Query highlightQuery = null;
-
- //-------------------------------------------------------------------------
- //-------------------------------------------------------------------------
-
+ public List<SearchComponent> components;
+
+ //////////////////////////////////////////////////////////
+ //////////////////////////////////////////////////////////
+ //// Distributed Search section
+ //////////////////////////////////////////////////////////
+ //////////////////////////////////////////////////////////
+
+ public static final String FIELD_SORT_VALUES = "fsv";
+ public static final String SHARDS = "shards";
+ public static final String IDS = "ids";
+
+ /***
+ public static final String NUMDOCS = "nd";
+ public static final String DOCFREQS = "tdf";
+ public static final String TERMS = "terms";
+ public static final String EXTRACT_QUERY_TERMS = "eqt";
+ public static final String LOCAL_SHARD = "local";
+ public static final String DOC_QUERY = "dq";
+ ***/
+
+ public static int STAGE_START = 0;
+ public static int STAGE_PARSE_QUERY = 1000;
+ public static int STAGE_EXECUTE_QUERY = 2000;
+ public static int STAGE_GET_FIELDS = 3000;
+ public static int STAGE_DONE = Integer.MAX_VALUE;
+
+ public int stage; // What stage is this current request at?
+
+
+ public String[] shards;
+ public List<ShardRequest> outgoing; // requests to be sent
+ public List<ShardRequest> finished; // requests that have received responses from all shards
+
+
+ public int getShardNum(String shard) {
+ for (int i=0; i<shards.length; i++) {
+ if (shards[i]==shard || shards[i].equals(shard)) return i;
+ }
+ return -1;
+ }
+
+ public void addRequest(SearchComponent me, ShardRequest sreq) {
+ outgoing.add(sreq);
+ if ((sreq.purpose & ShardRequest.PURPOSE_PRIVATE)==0) {
+ // if this isn't a private request, let other components modify it.
+ for (SearchComponent component : components) {
+ if (component != me) {
+ component.modifyRequest(this, me, sreq);
+ }
+ }
+ }
+ }
+
+ public GlobalCollectionStat globalCollectionStat;
+
+ Map<Object, ShardDoc> resultIds;
+ // Maps uniqueKeyValue to ShardDoc, which may be used to
+ // determine order of the doc or uniqueKey in the final
+ // returned sequence.
+ // Only valid after STAGE_EXECUTE_QUERY has completed.
+
+
+ /* private... components that don't own these shouldn't use them */
+ SolrDocumentList _responseDocs;
+ FacetInfo _facetInfo;
+
/**
* Utility function to add debugging info. This will make sure a valid
* debugInfo exists before adding to it.
@@ -70,7 +144,7 @@
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
-
+
public boolean isDebug() {
return debug;
}
@@ -174,4 +248,17 @@
public void setTimer(RTimer timer) {
this.timer = timer;
}
+
+
+ public static class GlobalCollectionStat {
+ public final long numDocs;
+
+ public final Map<String, Long> dfMap;
+
+ public GlobalCollectionStat(int numDocs, Map<String, Long> dfMap) {
+ this.numDocs = numDocs;
+ this.dfMap = dfMap;
+ }
+ }
+
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchComponent.java?rev=631357&r1=631356&r2=631357&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/SearchComponent.java Tue Feb 26 11:47:07 2008
@@ -36,11 +36,31 @@
*/
public abstract class SearchComponent implements SolrInfoMBean, NamedListInitializedPlugin
{
- public abstract void prepare( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException, ParseException;
- public abstract void process( SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException;
+ public abstract void prepare(ResponseBuilder rb) throws IOException;
+ public abstract void process(ResponseBuilder rb) throws IOException;
+
+ /** Process for a distributed search.
+ * @returns the next stage for this component */
+ public int distributedProcess(ResponseBuilder rb) throws IOException {
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ /** Called after another component adds a request */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
+ }
+
+ /** Called after all responses for a single request were received */
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ }
+
+ /** Called after all responses have been received for this stage.
+ * Useful when different requests are sent to each shard.
+ */
+ public void finishStage(ResponseBuilder rb) {
+ }
+
//////////////////////// NamedListInitializedPlugin methods //////////////////////
-
public void init( NamedList args )
{
// By default do nothing