You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-dev@lucene.apache.org by Noble Paul നോബിള് नोब्ळ् <no...@gmail.com> on 2008/07/06 07:05:51 UTC
Re: svn commit: r674249 - in /lucene/solr/trunk: ./ client/java/solrj/src/org/apache/solr/client/solrj/ client/java/solrj/test/org/apache/solr/client/solrj/ src/java/org/apache/solr/common/params/ src/java/org/apache/solr/handler/ src/java/org/apache
In SolrQuery all the set/add methods return SolrQuery to facilitate
method chaining.
Let us make
public void setTimeAllowed(Integer milliseconds)
to
public SolrQuery setTimeAllowed(Integer milliseconds)
--Noble
On Sun, Jul 6, 2008 at 6:46 AM, <yo...@apache.org> wrote:
> Author: yonik
> Date: Sat Jul 5 18:16:12 2008
> New Revision: 674249
>
> URL: http://svn.apache.org/viewvc?rev=674249&view=rev
> Log:
> SOLR-502: Add search timeout support
>
> Modified:
> lucene/solr/trunk/CHANGES.txt
> lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrQuery.java
> lucene/solr/trunk/client/java/solrj/test/org/apache/solr/client/solrj/SolrQueryTest.java
> lucene/solr/trunk/src/java/org/apache/solr/common/params/CommonParams.java
> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java
> lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
> lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java
> lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
>
> Modified: lucene/solr/trunk/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/CHANGES.txt (original)
> +++ lucene/solr/trunk/CHANGES.txt Sat Jul 5 18:16:12 2008
> @@ -305,6 +305,8 @@
>
> 57. SOLR-14: Add preserveOriginal flag to WordDelimiterFilter
> (Geoffrey Young, Trey Hyde, Ankur Madnani, yonik)
> +
> +58. SOLR-502: Add search timeout support. (Sean Timm via yonik)
>
> Changes in runtime behavior
> 1. SOLR-559: use Lucene updateDocument, deleteDocuments methods. This
>
> Modified: lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrQuery.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrQuery.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrQuery.java (original)
> +++ lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrQuery.java Sat Jul 5 18:16:12 2008
> @@ -464,6 +464,30 @@
> return q;
> }
>
> + /**
> + * Set the maximum time allowed for this query. If the query takes more time
> + * than the specified milliseconds, a timeout occurs and partial (or no)
> + * results may be returned.
> + *
> + * If given Long is null, then this parameter is removed from the request
> + *
> + *@param milliseconds the time in milliseconds allowed for this query
> + */
> + public void setTimeAllowed(Integer milliseconds) {
> + if (milliseconds == null) {
> + this.remove(CommonParams.TIME_ALLOWED);
> + } else {
> + this.set(CommonParams.TIME_ALLOWED, milliseconds);
> + }
> + }
> +
> + /**
> + * Get the maximum time allowed for this query.
> + */
> + public Integer getTimeAllowed() {
> + return this.getInt(CommonParams.TIME_ALLOWED);
> + }
> +
> ///////////////////////
> // Utility functions
> ///////////////////////
>
> Modified: lucene/solr/trunk/client/java/solrj/test/org/apache/solr/client/solrj/SolrQueryTest.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/test/org/apache/solr/client/solrj/SolrQueryTest.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/client/java/solrj/test/org/apache/solr/client/solrj/SolrQueryTest.java (original)
> +++ lucene/solr/trunk/client/java/solrj/test/org/apache/solr/client/solrj/SolrQueryTest.java Sat Jul 5 18:16:12 2008
> @@ -79,6 +79,13 @@
>
> // check to see that the removes are properly clearing the cgi params
> Assert.assertEquals(q.toString(), "q=dog");
> +
> + //Add time allowed param
> + q.setTimeAllowed(1000);
> + Assert.assertEquals((Integer)1000, q.getTimeAllowed() );
> + //Adding a null should remove it
> + q.setTimeAllowed(null);
> + Assert.assertEquals(null, q.getTimeAllowed() );
>
> System.out.println(q);
> }
>
> Modified: lucene/solr/trunk/src/java/org/apache/solr/common/params/CommonParams.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/common/params/CommonParams.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/src/java/org/apache/solr/common/params/CommonParams.java (original)
> +++ lucene/solr/trunk/src/java/org/apache/solr/common/params/CommonParams.java Sat Jul 5 18:16:12 2008
> @@ -78,7 +78,11 @@
> * will be used for all of them.
> */
> public static final String STREAM_CONTENTTYPE = "stream.contentType";
> -
> +
> + /**
> + * Timeout value in milliseconds. If not set, or the value is <= 0, there is no timeout.
> + */
> + public static final String TIME_ALLOWED = "timeAllowed";
>
> /** 'true' if the header should include the handler name */
> public static final String HEADER_ECHO_HANDLER = "echoHandler";
>
> Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerBase.java Sat Jul 5 18:16:12 2008
> @@ -18,6 +18,7 @@
> package org.apache.solr.handler;
>
> import org.apache.solr.common.SolrException;
> +import org.apache.solr.common.SolrDocumentList;
> import org.apache.solr.common.params.SolrParams;
> import org.apache.solr.common.util.NamedList;
> import org.apache.solr.common.util.SimpleOrderedMap;
> @@ -26,6 +27,7 @@
> import org.apache.solr.request.SolrQueryRequest;
> import org.apache.solr.request.SolrQueryResponse;
> import org.apache.solr.request.SolrRequestHandler;
> +import org.apache.solr.search.DocSet;
> import org.apache.solr.util.SolrPluginUtils;
> import org.apache.lucene.queryParser.ParseException;
>
> @@ -41,6 +43,7 @@
> // acceptable every million requests or so?
> volatile long numRequests;
> volatile long numErrors;
> + volatile long numTimeouts;
> protected NamedList initArgs = null;
> protected SolrParams defaults;
> protected SolrParams appends;
> @@ -123,6 +126,12 @@
> try {
> U.setDefaults(req,defaults,appends,invariants);
> handleRequestBody( req, rsp );
> + // count timeouts
> + boolean timedOut = (Boolean)rsp.getResponseHeader().get("partialResults") == null ? false : (Boolean)rsp.getResponseHeader().get("partialResults");
> + if( timedOut ) {
> + numTimeouts++;
> + rsp.setHttpCaching(false);
> + }
> } catch (Exception e) {
> SolrException.log(SolrCore.log,e);
> if (e instanceof ParseException) {
> @@ -158,6 +167,7 @@
> NamedList lst = new SimpleOrderedMap();
> lst.add("requests", numRequests);
> lst.add("errors", numErrors);
> + lst.add("timeouts", numTimeouts);
> lst.add("avgTimePerRequest", (float) totalTime / (float) this.numRequests);
> lst.add("avgRequestsPerSecond", (float) numRequests*1000 / (float)(System.currentTimeMillis()-handlerStart));
> return lst;
>
> Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java Sat Jul 5 18:16:12 2008
> @@ -61,7 +61,7 @@
> SolrQueryRequest req = rb.req;
> SolrQueryResponse rsp = rb.rsp;
> SolrParams params = req.getParams();
> -
> +
> // Set field flags
> String fl = params.get(CommonParams.FL);
> int fieldFlags = 0;
> @@ -120,6 +120,9 @@
> SolrIndexSearcher searcher = req.getSearcher();
> SolrParams params = req.getParams();
>
> + // -1 as flag if not set.
> + long timeAllowed = (long)params.getInt( CommonParams.TIME_ALLOWED, -1 );
> +
> // Optional: This could also be implemented by the top-level searcher sending
> // a filter that lists the ids... that would be transparent to
> // the request handler, but would be more expensive (and would preserve score
> @@ -151,20 +154,11 @@
> return;
> }
>
> - if( rb.isNeedDocSet() ) {
> - rb.setResults( searcher.getDocListAndSet(
> - rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
> - rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
> - rb.getFieldFlags() ) );
> - }
> - else {
> - DocListAndSet results = new DocListAndSet();
> - results.docList = searcher.getDocList(
> - rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort(),
> - rb.getSortSpec().getOffset(), rb.getSortSpec().getCount(),
> - rb.getFieldFlags() );
> - rb.setResults( results );
> - }
> + SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
> + cmd.setTimeAllowed(timeAllowed);
> + SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
> + searcher.search(result,cmd);
> + rb.setResult( result );
>
> rsp.add("response",rb.getResults().docList);
> rsp.getToLog().add("hits", rb.getResults().docList.size());
> @@ -432,7 +426,6 @@
> if (maxScore!=null) responseDocs.setMaxScore(maxScore);
> responseDocs.setNumFound(numFound);
> responseDocs.setStart(ss.getOffset());
> -
> // size appropriately
> for (int i=0; i<resultSize; i++) responseDocs.add(null);
>
> @@ -504,7 +497,6 @@
> // could/should bypass middlemen (like retrieving stored fields)
> // TODO: merge fsv to if requested
>
> -
> if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
> boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
>
>
> Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java (original)
> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/ResponseBuilder.java Sat Jul 5 18:16:12 2008
> @@ -27,6 +27,7 @@
> import org.apache.solr.search.DocListAndSet;
> import org.apache.solr.search.QParser;
> import org.apache.solr.search.SortSpec;
> +import org.apache.solr.search.SolrIndexSearcher;
>
> import java.util.LinkedHashMap;
> import java.util.List;
> @@ -261,4 +262,29 @@
> }
> }
>
> + /**
> + * Creates a SolrIndexSearcher.QueryCommand from this
> + * ResponseBuilder. TimeAllowed is left unset.
> + */
> + public SolrIndexSearcher.QueryCommand getQueryCommand() {
> + SolrIndexSearcher.QueryCommand cmd = new SolrIndexSearcher.QueryCommand();
> + cmd.setQuery( getQuery() )
> + .setFilterList( getFilters() )
> + .setSort( getSortSpec().getSort() )
> + .setOffset( getSortSpec().getOffset() )
> + .setLen( getSortSpec().getCount() )
> + .setFlags( getFieldFlags() )
> + .setNeedDocSet( isNeedDocSet() );
> + return cmd;
> + }
> +
> + /**
> + * Sets results from a SolrIndexSearcher.QueryResult.
> + */
> + public void setResult( SolrIndexSearcher.QueryResult result ) {
> + setResults( result.getDocListAndSet() );
> + if( result.isPartialResults() ) {
> + rsp.getResponseHeader().add( "partialResults", Boolean.TRUE );
> + }
> + }
> }
>
> Modified: lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
> URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=674249&r1=674248&r2=674249&view=diff
> ==============================================================================
> --- lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
> +++ lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Jul 5 18:16:12 2008
> @@ -248,10 +248,16 @@
> nDocs = Math.min(oldnDocs,40);
> }
>
> - DocListAndSet ret = new DocListAndSet();
> int flags=NO_CHECK_QCACHE | key.nc_flags;
> -
> - newSearcher.getDocListC(ret, key.query, key.filters, null, key.sort, 0, nDocs, flags);
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(key.query)
> + .setFilterList(key.filters)
> + .setSort(key.sort)
> + .setLen(nDocs)
> + .setSupersetMaxDoc(nDocs)
> + .setFlags(flags);
> + QueryResult qr = new QueryResult();
> + newSearcher.getDocListC(qr,qc);
> return true;
> }
> }
> @@ -259,6 +265,11 @@
> }
> }
>
> + public QueryResult search(QueryResult qr, QueryCommand cmd) throws IOException {
> + getDocListC(qr,cmd);
> + return qr;
> + }
> +
> public Hits search(Query query, Filter filter, Sort sort) throws IOException {
> // todo - when Solr starts accepting filters, need to
> // change this conditional check (filter!=null) and create a new filter
> @@ -669,12 +680,15 @@
> * @throws IOException
> */
> public DocList getDocList(Query query, Query filter, Sort lsort, int offset, int len) throws IOException {
> - List<Query> filterList = null;
> - if (filter != null) {
> - filterList = new ArrayList<Query>(1);
> - filterList.add(filter);
> - }
> - return getDocList(query, filterList, lsort, offset, len, 0);
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocList();
> }
>
>
> @@ -696,59 +710,69 @@
> * @throws IOException
> */
> public DocList getDocList(Query query, List<Query> filterList, Sort lsort, int offset, int len, int flags) throws IOException {
> - DocListAndSet answer = new DocListAndSet();
> - getDocListC(answer,query,filterList,null,lsort,offset,len,flags);
> - return answer.docList;
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filterList)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setFlags(flags);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocList();
> }
>
> -
> private static final int NO_CHECK_QCACHE = 0x80000000;
> private static final int GET_DOCSET = 0x40000000;
> private static final int NO_CHECK_FILTERCACHE = 0x20000000;
>
> public static final int GET_SCORES = 0x01;
>
> - /** getDocList version that uses+populates query and filter caches.
> - * This should only be called using either filterList or filter, but not both.
> + /**
> + * getDocList version that uses+populates query and filter caches.
> + * In the event of a timeout, the cache is not populated.
> */
> - private void getDocListC(DocListAndSet out, Query query, List<Query> filterList, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
> + private void getDocListC(QueryResult qr, QueryCommand cmd) throws IOException {
> + // old parameters: DocListAndSet out, Query query, List<Query> filterList, DocSet filter, Sort lsort, int offset, int len, int flags, long timeAllowed, NamedList<Object> responseHeader
> + DocListAndSet out = new DocListAndSet();
> + qr.setDocListAndSet(out);
> QueryResultKey key=null;
> - int maxDocRequested = offset + len;
> + int maxDocRequested = cmd.getOffset() + cmd.getLen();
> // check for overflow, and check for # docs in index
> if (maxDocRequested < 0 || maxDocRequested > maxDoc()) maxDocRequested = maxDoc();
> int supersetMaxDoc= maxDocRequested;
> DocList superset;
>
> -
> // we can try and look up the complete query in the cache.
> // we can't do that if filter!=null though (we don't want to
> // do hashCode() and equals() for a big DocSet).
> - if (queryResultCache != null && filter==null) {
> + if (queryResultCache != null && cmd.getFilter()==null) {
> // all of the current flags can be reused during warming,
> // so set all of them on the cache key.
> - key = new QueryResultKey(query, filterList, lsort, flags);
> - if ((flags & NO_CHECK_QCACHE)==0) {
> + key = new QueryResultKey(cmd.getQuery(), cmd.getFilterList(), cmd.getSort(), cmd.getFlags());
> + if ((cmd.getFlags() & NO_CHECK_QCACHE)==0) {
> superset = (DocList)queryResultCache.get(key);
>
> if (superset != null) {
> // check that the cache entry has scores recorded if we need them
> - if ((flags & GET_SCORES)==0 || superset.hasScores()) {
> + if ((cmd.getFlags() & GET_SCORES)==0 || superset.hasScores()) {
> // NOTE: subset() returns null if the DocList has fewer docs than
> // requested
> - out.docList = superset.subset(offset,len);
> + out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
> }
> }
> if (out.docList != null) {
> // found the docList in the cache... now check if we need the docset too.
> // OPT: possible future optimization - if the doclist contains all the matches,
> // use it to make the docset instead of rerunning the query.
> - if (out.docSet==null && ((flags & GET_DOCSET)!=0) ) {
> - if (filterList==null) {
> - out.docSet = getDocSet(query);
> + if (out.docSet==null && ((cmd.getFlags() & GET_DOCSET)!=0) ) {
> + if (cmd.getFilterList()==null) {
> + out.docSet = getDocSet(cmd.getQuery());
> } else {
> - List<Query> newList = new ArrayList<Query>(filterList.size()+1);
> - newList.add(query);
> - newList.addAll(filterList);
> + List<Query> newList = new ArrayList<Query>(cmd.getFilterList()
> +.size()+1);
> + newList.add(cmd.getQuery());
> + newList.addAll(cmd.getFilterList());
> out.docSet = getDocSet(newList);
> }
> }
> @@ -778,9 +802,9 @@
>
> // check if we should try and use the filter cache
> boolean useFilterCache=false;
> - if ((flags & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && lsort != null && filterCache != null) {
> + if ((cmd.getFlags() & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && cmd.getSort() != null && filterCache != null) {
> useFilterCache=true;
> - SortField[] sfields = lsort.getSort();
> + SortField[] sfields = cmd.getSort().getSort();
> for (SortField sf : sfields) {
> if (sf.getType() == SortField.SCORE) {
> useFilterCache=false;
> @@ -794,41 +818,46 @@
> // for large filters that match few documents, this may be
> // slower than simply re-executing the query.
> if (out.docSet == null) {
> - out.docSet = getDocSet(query,filter);
> - DocSet bigFilt = getDocSet(filterList);
> + out.docSet = getDocSet(cmd.getQuery(),cmd.getFilter());
> + DocSet bigFilt = getDocSet(cmd.getFilterList());
> if (bigFilt != null) out.docSet = out.docSet.intersection(bigFilt);
> }
> // todo: there could be a sortDocSet that could take a list of
> // the filters instead of anding them first...
> // perhaps there should be a multi-docset-iterator
> - superset = sortDocSet(out.docSet,lsort,supersetMaxDoc);
> - out.docList = superset.subset(offset,len);
> + superset = sortDocSet(out.docSet,cmd.getSort(),supersetMaxDoc);
> + out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
> } else {
> // do it the normal way...
> - DocSet theFilt = filter!=null ? filter : getDocSet(filterList);
> -
> - if ((flags & GET_DOCSET)!=0) {
> - DocSet qDocSet = getDocListAndSetNC(out,query,theFilt,lsort,0,supersetMaxDoc,flags);
> + cmd.setSupersetMaxDoc(supersetMaxDoc);
> + if ((cmd.getFlags() & GET_DOCSET)!=0) {
> + DocSet qDocSet = getDocListAndSetNC(qr,cmd);
> // cache the docSet matching the query w/o filtering
> - if (filterCache!=null) filterCache.put(query,qDocSet);
> + if (filterCache!=null && !qr.isPartialResults()) filterCache.put(cmd.getQuery(),qDocSet);
> } else {
> - out.docList = getDocListNC(query,theFilt,lsort,0,supersetMaxDoc,flags);
> + getDocListNC(qr,cmd);
> + //Parameters: cmd.getQuery(),theFilt,cmd.getSort(),0,supersetMaxDoc,cmd.getFlags(),cmd.getTimeAllowed(),responseHeader);
> }
> superset = out.docList;
> - out.docList = superset.subset(offset,len);
> + out.docList = superset.subset(cmd.getOffset(),cmd.getLen());
> }
>
> // lastly, put the superset in the cache if the size is less than or equal
> // to queryResultMaxDocsCached
> - if (key != null && superset.size() <= queryResultMaxDocsCached) {
> + if (key != null && superset.size() <= queryResultMaxDocsCached && !qr.isPartialResults()) {
> queryResultCache.put(key, superset);
> }
> }
>
>
>
> - private DocList getDocListNC(Query query, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
> - int last = offset+len;
> + private void getDocListNC(QueryResult qr,QueryCommand cmd) throws IOException {
> + //Parameters: cmd.getQuery(),theFilt,cmd.getSort(),0,supersetMaxDoc,cmd.getFlags(),cmd.getTimeAllowed(),responseHeader);
> + //Query query, DocSet filter, Sort lsort, int offset, int len, int flags, long timeAllowed, NamedList<Object> responseHeader
> + DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
> + final long timeAllowed = cmd.getTimeAllowed();
> + int len = cmd.getSupersetMaxDoc();
> + int last = len;
> if (last < 0 || last > maxDoc()) last=maxDoc();
> final int lastDocRequested = last;
> int nDocsReturned;
> @@ -837,7 +866,7 @@
> int[] ids;
> float[] scores;
>
> - query = QueryUtils.makeQueryable(query);
> + Query query = QueryUtils.makeQueryable(cmd.getQuery());
>
> // handle zero case...
> if (lastDocRequested<=0) {
> @@ -845,44 +874,62 @@
> final float[] topscore = new float[] { Float.NEGATIVE_INFINITY };
> final int[] numHits = new int[1];
>
> - searcher.search(query, new HitCollector() {
> + HitCollector hc = new HitCollector() {
> public void collect(int doc, float score) {
> if (filt!=null && !filt.exists(doc)) return;
> numHits[0]++;
> if (score > topscore[0]) topscore[0]=score;
> }
> + };
> + if( timeAllowed > 0 ) {
> + hc = new TimeLimitedCollector( hc, timeAllowed );
> + }
> + try {
> + searcher.search(query, hc );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> nDocsReturned=0;
> ids = new int[nDocsReturned];
> scores = new float[nDocsReturned];
> totalHits = numHits[0];
> maxScore = totalHits>0 ? topscore[0] : 0.0f;
> - } else if (lsort != null) {
> + } else if (cmd.getSort() != null) {
> // can't use TopDocs if there is a sort since it
> // will do automatic score normalization.
> // NOTE: this changed late in Lucene 1.9
>
> final DocSet filt = filter;
> final int[] numHits = new int[1];
> - final FieldSortedHitQueue hq = new FieldSortedHitQueue(reader, lsort.getSort(), offset+len);
> + final FieldSortedHitQueue hq = new FieldSortedHitQueue(reader, cmd.getSort().getSort(), len);
>
> - searcher.search(query, new HitCollector() {
> + HitCollector hc = new HitCollector() {
> public void collect(int doc, float score) {
> if (filt!=null && !filt.exists(doc)) return;
> numHits[0]++;
> hq.insert(new FieldDoc(doc, score));
> }
> + };
> + if( timeAllowed > 0 ) {
> + hc = new TimeLimitedCollector( hc, timeAllowed );
> + }
> + try {
> + searcher.search(query, hc );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> totalHits = numHits[0];
> maxScore = totalHits>0 ? hq.getMaxScore() : 0.0f;
>
> nDocsReturned = hq.size();
> ids = new int[nDocsReturned];
> - scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> + scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> for (int i = nDocsReturned -1; i >= 0; i--) {
> FieldDoc fieldDoc = (FieldDoc)hq.pop();
> // fillFields is the point where score normalization happens
> @@ -898,7 +945,7 @@
> final DocSet filt = filter;
> final ScorePriorityQueue hq = new ScorePriorityQueue(lastDocRequested);
> final int[] numHits = new int[1];
> - searcher.search(query, new HitCollector() {
> + HitCollector hc = new HitCollector() {
> float minScore=Float.NEGATIVE_INFINITY; // minimum score in the priority queue
> public void collect(int doc, float score) {
> if (filt!=null && !filt.exists(doc)) return;
> @@ -911,13 +958,22 @@
> minScore = ((ScoreDoc)hq.top()).score;
> }
> }
> + };
> + if( timeAllowed > 0 ) {
> + hc = new TimeLimitedCollector( hc, timeAllowed );
> + }
> + try {
> + searcher.search(query, hc );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> totalHits = numHits[0];
> nDocsReturned = hq.size();
> ids = new int[nDocsReturned];
> - scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> + scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> ScoreDoc sdoc =null;
> for (int i = nDocsReturned -1; i >= 0; i--) {
> sdoc = (ScoreDoc)hq.pop();
> @@ -928,9 +984,9 @@
> }
>
>
> - int sliceLen = Math.min(lastDocRequested,nDocsReturned) - offset;
> + int sliceLen = Math.min(lastDocRequested,nDocsReturned);
> if (sliceLen < 0) sliceLen=0;
> - return new DocSlice(offset,sliceLen,ids,scores,totalHits,maxScore);
> + qr.setDocList(new DocSlice(0,sliceLen,ids,scores,totalHits,maxScore));
>
>
>
> @@ -987,8 +1043,10 @@
>
> // the DocSet returned is for the query only, without any filtering... that way it may
> // be cached if desired.
> - private DocSet getDocListAndSetNC(DocListAndSet out, Query query, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
> - int last = offset+len;
> + private DocSet getDocListAndSetNC(QueryResult qr,QueryCommand cmd) throws IOException {
> + int len = cmd.getSupersetMaxDoc();
> + DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
> + int last = len;
> if (last < 0 || last > maxDoc()) last=maxDoc();
> final int lastDocRequested = last;
> int nDocsReturned;
> @@ -997,8 +1055,9 @@
> int[] ids;
> float[] scores;
> final DocSetHitCollector setHC = new DocSetHitCollector(HASHSET_INVERSE_LOAD_FACTOR, HASHDOCSET_MAXSIZE, maxDoc());
> + final HitCollector hitCollector = ( cmd.getTimeAllowed() > 0 ) ? new TimeLimitedCollector( setHC, cmd.getTimeAllowed() ) : setHC;
>
> - query = QueryUtils.makeQueryable(query);
> + Query query = QueryUtils.makeQueryable(cmd.getQuery());
>
> // TODO: perhaps unify getDocListAndSetNC and getDocListNC without imposing a significant performance hit
>
> @@ -1018,46 +1077,58 @@
> final float[] topscore = new float[] { Float.NEGATIVE_INFINITY };
> final int[] numHits = new int[1];
>
> - searcher.search(query, new HitCollector() {
> - public void collect(int doc, float score) {
> - setHC.collect(doc,score);
> - if (filt!=null && !filt.exists(doc)) return;
> - numHits[0]++;
> - if (score > topscore[0]) topscore[0]=score;
> + try {
> + searcher.search(query, new HitCollector() {
> + public void collect(int doc, float score) {
> + hitCollector.collect(doc,score);
> + if (filt!=null && !filt.exists(doc)) return;
> + numHits[0]++;
> + if (score > topscore[0]) topscore[0]=score;
> + }
> }
> + );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> nDocsReturned=0;
> ids = new int[nDocsReturned];
> scores = new float[nDocsReturned];
> totalHits = numHits[0];
> maxScore = totalHits>0 ? topscore[0] : 0.0f;
> - } else if (lsort != null) {
> + } else if (cmd.getSort() != null) {
> // can't use TopDocs if there is a sort since it
> // will do automatic score normalization.
> // NOTE: this changed late in Lucene 1.9
>
> final DocSet filt = filter;
> final int[] numHits = new int[1];
> - final FieldSortedHitQueue hq = new FieldSortedHitQueue(reader, lsort.getSort(), offset+len);
> + final FieldSortedHitQueue hq = new FieldSortedHitQueue(reader, cmd.getSort().getSort(), len);
>
> - searcher.search(query, new HitCollector() {
> - public void collect(int doc, float score) {
> - setHC.collect(doc,score);
> - if (filt!=null && !filt.exists(doc)) return;
> - numHits[0]++;
> - hq.insert(new FieldDoc(doc, score));
> + try {
> + searcher.search(query, new HitCollector() {
> + public void collect(int doc, float score) {
> + hitCollector.collect(doc,score);
> + if (filt!=null && !filt.exists(doc)) return;
> + numHits[0]++;
> + hq.insert(new FieldDoc(doc, score));
> + }
> }
> + );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> totalHits = numHits[0];
> maxScore = totalHits>0 ? hq.getMaxScore() : 0.0f;
>
> nDocsReturned = hq.size();
> ids = new int[nDocsReturned];
> - scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> + scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> for (int i = nDocsReturned -1; i >= 0; i--) {
> FieldDoc fieldDoc = (FieldDoc)hq.pop();
> // fillFields is the point where score normalization happens
> @@ -1073,25 +1144,31 @@
> final DocSet filt = filter;
> final ScorePriorityQueue hq = new ScorePriorityQueue(lastDocRequested);
> final int[] numHits = new int[1];
> - searcher.search(query, new HitCollector() {
> - float minScore=Float.NEGATIVE_INFINITY; // minimum score in the priority queue
> - public void collect(int doc, float score) {
> - setHC.collect(doc,score);
> - if (filt!=null && !filt.exists(doc)) return;
> - if (numHits[0]++ < lastDocRequested || score >= minScore) {
> - // if docs are always delivered in order, we could use "score>minScore"
> - // but might BooleanScorer14 might still be used and deliver docs out-of-order?
> - hq.insert(new ScoreDoc(doc, score));
> - minScore = ((ScoreDoc)hq.top()).score;
> + try {
> + searcher.search(query, new HitCollector() {
> + float minScore=Float.NEGATIVE_INFINITY; // minimum score in the priority queue
> + public void collect(int doc, float score) {
> + hitCollector.collect(doc,score);
> + if (filt!=null && !filt.exists(doc)) return;
> + if (numHits[0]++ < lastDocRequested || score >= minScore) {
> + // if docs are always delivered in order, we could use "score>minScore"
> + // but might BooleanScorer14 might still be used and deliver docs out-of-order?
> + hq.insert(new ScoreDoc(doc, score));
> + minScore = ((ScoreDoc)hq.top()).score;
> + }
> }
> }
> + );
> + }
> + catch( TimeLimitedCollector.TimeExceededException x ) {
> + log.warning( "Query: " + query + "; " + x.getMessage() );
> + qr.setPartialResults(true);
> }
> - );
>
> totalHits = numHits[0];
> nDocsReturned = hq.size();
> ids = new int[nDocsReturned];
> - scores = (flags&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> + scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
> ScoreDoc sdoc =null;
> for (int i = nDocsReturned -1; i >= 0; i--) {
> sdoc = (ScoreDoc)hq.pop();
> @@ -1102,11 +1179,12 @@
> }
>
>
> - int sliceLen = Math.min(lastDocRequested,nDocsReturned) - offset;
> + int sliceLen = Math.min(lastDocRequested,nDocsReturned);
> if (sliceLen < 0) sliceLen=0;
> - out.docList = new DocSlice(offset,sliceLen,ids,scores,totalHits,maxScore);
> +
> + qr.setDocList(new DocSlice(0,sliceLen,ids,scores,totalHits,maxScore));
> DocSet qDocSet = setHC.getDocSet();
> - out.docSet = filter==null ? qDocSet : qDocSet.intersection(filter);
> + qr.setDocSet(filter==null ? qDocSet : qDocSet.intersection(filter));
> return qDocSet;
> }
>
> @@ -1126,9 +1204,15 @@
> * @throws IOException
> */
> public DocList getDocList(Query query, DocSet filter, Sort lsort, int offset, int len) throws IOException {
> - DocListAndSet answer = new DocListAndSet();
> - getDocListC(answer,query,null,filter,lsort,offset,len,0);
> - return answer.docList;
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilter(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocList();
> }
>
> /**
> @@ -1152,9 +1236,16 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, Query filter, Sort lsort, int offset, int len) throws IOException {
> - List<Query> filterList = buildQueryList(filter);
> - return getDocListAndSet(query, filterList, lsort, offset, len);
> -
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> }
>
> /**
> @@ -1179,22 +1270,19 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, Query filter, Sort lsort, int offset, int len, int flags) throws IOException {
> - List<Query> filterList = buildQueryList(filter);
> - return getDocListAndSet(query, filterList, lsort, offset, len, flags);
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setFlags(flags)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> }
>
> - /**
> - * A simple utility method for to build a filterList from a query
> - * @param filter
> - */
> - private List<Query> buildQueryList(Query filter) {
> - List<Query> filterList = null;
> - if (filter != null) {
> - filterList = new ArrayList<Query>(2);
> - filterList.add(filter);
> - }
> - return filterList;
> - }
>
> /**
> * Returns documents matching both <code>query</code> and the intersection
> @@ -1219,9 +1307,16 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, List<Query> filterList, Sort lsort, int offset, int len) throws IOException {
> - DocListAndSet ret = new DocListAndSet();
> - getDocListC(ret,query,filterList,null,lsort,offset,len,GET_DOCSET);
> - return ret;
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filterList)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> }
>
> /**
> @@ -1248,9 +1343,17 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, List<Query> filterList, Sort lsort, int offset, int len, int flags) throws IOException {
> - DocListAndSet ret = new DocListAndSet();
> - getDocListC(ret,query,filterList,null,lsort,offset,len, flags |= GET_DOCSET);
> - return ret;
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilterList(filterList)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setFlags(flags)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> }
>
> /**
> @@ -1269,9 +1372,16 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, DocSet filter, Sort lsort, int offset, int len) throws IOException {
> - DocListAndSet ret = new DocListAndSet();
> - getDocListC(ret,query,null,filter,lsort,offset,len,GET_DOCSET);
> - return ret;
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilter(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> }
>
> /**
> @@ -1296,10 +1406,18 @@
> * @throws IOException
> */
> public DocListAndSet getDocListAndSet(Query query, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException {
> - DocListAndSet ret = new DocListAndSet();
> - getDocListC(ret,query,null,filter,lsort,offset,len, flags |= GET_DOCSET);
> - return ret;
> - }
> + QueryCommand qc = new QueryCommand();
> + qc.setQuery(query)
> + .setFilter(filter)
> + .setSort(lsort)
> + .setOffset(offset)
> + .setLen(len)
> + .setFlags(flags)
> + .setNeedDocSet(true);
> + QueryResult qr = new QueryResult();
> + search(qr,qc);
> + return qr.getDocListAndSet();
> + }
>
> protected DocList sortDocSet(DocSet set, Sort sort, int nDocs) throws IOException {
> final FieldSortedHitQueue hq =
> @@ -1473,9 +1591,154 @@
> if (registerTime!=0) lst.add("registeredAt", new Date(registerTime));
> return lst;
> }
> -}
>
> + /**
> + * A query request command to avoid having to change the method signatures
> + * if we want to pass additional information to the searcher.
> + */
> + public static class QueryCommand {
> + private Query query;
> + private List<Query> filterList;
> + private DocSet filter;
> + private Sort sort;
> + private int offset;
> + private int len;
> + private int supersetMaxDoc;
> + private int flags;
> + private long timeAllowed = -1;
> + private boolean needDocSet;
> +
> + public Query getQuery() { return query; }
> + public QueryCommand setQuery(Query query) {
> + this.query = query;
> + return this;
> + }
> +
> + public List<Query> getFilterList() { return filterList; }
> + /**
> + * @throws IllegalArgumentException if filter is not null.
> + */
> + public QueryCommand setFilterList(List<Query> filterList) {
> + if( filter != null ) {
> + throw new IllegalArgumentException( "Either filter or filterList may be set in the QueryCommand, but not both." );
> + }
> + this.filterList = filterList;
> + return this;
> + }
> + /**
> + * A simple setter to build a filterList from a query
> + * @throws IllegalArgumentException if filter is not null.
> + */
> + public QueryCommand setFilterList(Query f) {
> + if( filter != null ) {
> + throw new IllegalArgumentException( "Either filter or filterList may be set in the QueryCommand, but not both." );
> + }
> + filterList = null;
> + if (f != null) {
> + filterList = new ArrayList<Query>(2);
> + filterList.add(f);
> + }
> + return this;
> + }
> +
> + public DocSet getFilter() { return filter; }
> + /**
> + * @throws IllegalArgumentException if filterList is not null.
> + */
> + public QueryCommand setFilter(DocSet filter) {
> + if( filterList != null ) {
> + throw new IllegalArgumentException( "Either filter or filterList may be set in the QueryCommand, but not both." );
> + }
> + this.filter = filter;
> + return this;
> + }
> +
> + public Sort getSort() { return sort; }
> + public QueryCommand setSort(Sort sort) {
> + this.sort = sort;
> + return this;
> + }
> +
> + public int getOffset() { return offset; }
> + public QueryCommand setOffset(int offset) {
> + this.offset = offset;
> + return this;
> + }
> +
> + public int getLen() { return len; }
> + public QueryCommand setLen(int len) {
> + this.len = len;
> + return this;
> + }
> +
> + public int getSupersetMaxDoc() { return supersetMaxDoc; }
> + public QueryCommand setSupersetMaxDoc(int supersetMaxDoc) {
> + this.supersetMaxDoc = supersetMaxDoc;
> + return this;
> + }
> +
> + public int getFlags() {
> + return flags;
> + }
>
> + public QueryCommand replaceFlags(int flags) {
> + this.flags = flags;
> + return this;
> + }
> +
> + public QueryCommand setFlags(int flags) {
> + this.flags |= flags;
> + return this;
> + }
> +
> + public QueryCommand clearFlags(int flags) {
> + this.flags &= ~flags;
> + return this;
> + }
> +
> + public long getTimeAllowed() { return timeAllowed; }
> + public QueryCommand setTimeAllowed(long timeAllowed) {
> + this.timeAllowed = timeAllowed;
> + return this;
> + }
> +
> + public boolean isNeedDocSet() { return (flags & GET_DOCSET) != 0; }
> + public QueryCommand setNeedDocSet(boolean needDocSet) {
> + return needDocSet ? setFlags(GET_DOCSET) : clearFlags(GET_DOCSET);
> + }
> + }
> +
> + /**
> + * The result of a search.
> + */
> + public static class QueryResult {
> + private boolean partialResults;
> + private DocListAndSet docListAndSet;
> +
> + public DocList getDocList() { return docListAndSet.docList; }
> + public void setDocList(DocList list) {
> + if( docListAndSet == null ) {
> + docListAndSet = new DocListAndSet();
> + }
> + docListAndSet.docList = list;
> + }
> +
> + public DocSet getDocSet() { return docListAndSet.docSet; }
> + public void setDocSet(DocSet set) {
> + if( docListAndSet == null ) {
> + docListAndSet = new DocListAndSet();
> + }
> + docListAndSet.docSet = set;
> + }
> +
> + public boolean isPartialResults() { return partialResults; }
> + public void setPartialResults(boolean partialResults) { this.partialResults = partialResults; }
> +
> + public void setDocListAndSet( DocListAndSet listSet ) { docListAndSet = listSet; }
> + public DocListAndSet getDocListAndSet() { return docListAndSet; }
> + }
> +
> +}
>
> // Lucene's HitQueue isn't public, so here is our own.
> final class ScorePriorityQueue extends PriorityQueue {
> @@ -1492,4 +1755,3 @@
> }
>
Re: svn commit: r674249 - in /lucene/solr/trunk: ./ client/java/solrj/src/org/apache/solr/client/solrj/ client/java/solrj/test/org/apache/solr/client/solrj/ src/java/org/apache/solr/common/params/ src/java/org/apache/solr/handler/ src/java/org/apache
Posted by Yonik Seeley <yo...@apache.org>.
On Sun, Jul 6, 2008 at 1:05 AM, Noble Paul നോബിള് नोब्ळ्
<no...@gmail.com> wrote:
> In SolrQuery all the set/add methods return SolrQuery to facilitate
> method chaining.
> Let us make
> public void setTimeAllowed(Integer milliseconds)
> to
> public SolrQuery setTimeAllowed(Integer milliseconds)
Done.
-Yonik