You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/21 12:16:20 UTC
svn commit: r1559973 - in /lucene/dev/branches/lucene5376/lucene/server/src:
java/org/apache/lucene/server/ java/org/apache/lucene/server/handlers/
test/org/apache/lucene/server/
Author: mikemccand
Date: Tue Jan 21 11:16:20 2014
New Revision: 1559973
URL: http://svn.apache.org/r1559973
Log:
LUCENE-5376: add search timeout (TimeLimitingCollector)
Modified:
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java?rev=1559973&r1=1559972&r2=1559973&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java Tue Jan 21 11:16:20 2014
@@ -41,12 +41,12 @@ import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.server.handlers.DocHandler;
import org.apache.lucene.server.handlers.Handler;
import org.apache.lucene.server.plugins.Plugin;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.NamedThreadFactory;
-
import net.minidev.json.JSONObject;
import net.minidev.json.JSONValue;
import net.minidev.json.parser.ParseException;
@@ -257,6 +257,12 @@ public class GlobalState implements Clos
//System.out.println("GlobalState.close");
IOUtils.close(indices.values());
indexService.shutdown();
+ TimeLimitingCollector.getGlobalTimerThread().stopTimer();
+ try {
+ TimeLimitingCollector.getGlobalTimerThread().join();
+ } catch (InterruptedException ie) {
+ throw new RuntimeException(ie);
+ }
}
/** Load any plugins. */
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java?rev=1559973&r1=1559972&r2=1559973&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java Tue Jan 21 11:16:20 2014
@@ -95,6 +95,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
@@ -417,7 +418,8 @@ public class SearchHandler extends Handl
new StructType(
new Param("doMaxScore", "Compute the max score across all hits (costs added CPU).", new BooleanType(), false),
new Param("doDocScores", "Compute the doc score for each collected (costs added CPU).", new BooleanType(), false),
- new Param("fields", "Fields to sort on.", SORT_TYPE)))
+ new Param("fields", "Fields to sort on.", SORT_TYPE))),
+ new Param("timeoutSec", "Maximum number of seconds spent on each collection phase; note that for multi-pass searches (e.g. query-time grouping), this timeout applies to each phase.", new FloatType())
);
@Override
public String getTopDoc() {
@@ -2147,6 +2149,19 @@ public class SearchHandler extends Handl
false);
}
+ long timeoutMS;
+ Collector c2;
+ if (r.hasParam("timeoutSec")) {
+ timeoutMS = (long) (r.getFloat("timeoutSec") * 1000);
+ if (timeoutMS <= 0) {
+ r.fail("timeoutSec", "must be > 0 msec");
+ }
+ c2 = new TimeLimitingCollector(c, TimeLimitingCollector.getGlobalCounter(), timeoutMS);
+ } else {
+ c2 = c;
+ timeoutMS = -1;
+ }
+
// nocommit can we do better? sometimes downgrade
// to DDQ not DS?
@@ -2192,9 +2207,18 @@ public class SearchHandler extends Handl
};
// Fills in facetResults as a side-effect:
- ds.search(ddq, c);
+ try {
+ ds.search(ddq, c2);
+ } catch (TimeLimitingCollector.TimeExceededException tee) {
+ result.put("hitTimeout", true);
+ }
+
} else {
- s.searcher.search(ddq, c);
+ try {
+ s.searcher.search(ddq, c2);
+ } catch (TimeLimitingCollector.TimeExceededException tee) {
+ result.put("hitTimeout", true);
+ }
}
diagnostics.put("firstPassSearchMS", ((System.nanoTime()-searchStartTime)/1000000.0));
@@ -2213,8 +2237,18 @@ public class SearchHandler extends Handl
grouping.getBoolean("doMaxScore"),
true);
long t0 = System.nanoTime();
+
+ c = c3;
//((MyIndexSearcher) s).search(w, c3);
- s.searcher.search(ddq, c3);
+ if (timeoutMS > 0) {
+ c = new TimeLimitingCollector(c, TimeLimitingCollector.getGlobalCounter(), timeoutMS);
+ }
+
+ try {
+ s.searcher.search(ddq, c);
+ } catch (TimeLimitingCollector.TimeExceededException tee) {
+ result.put("hitTimeout", true);
+ }
diagnostics.put("secondPassSearchMS", ((System.nanoTime()-t0)/1000000));
groups = c3.getTopGroups(0);
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java?rev=1559973&r1=1559972&r2=1559973&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java Tue Jan 21 11:16:20 2014
@@ -444,6 +444,19 @@ public abstract class ServerBaseTestCase
return o;
}
+ protected boolean hasParam(Object o, String path) {
+ try {
+ get(o, path);
+ return true;
+ } catch (IllegalArgumentException iae) {
+ return false;
+ }
+ }
+
+ protected boolean hasParam(String path) {
+ return hasParam(lastResult, path);
+ }
+
protected static String getString(Object o, String path) {
return (String) get(o, path);
}
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java?rev=1559973&r1=1559972&r2=1559973&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java Tue Jan 21 11:16:20 2014
@@ -22,6 +22,8 @@ import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -248,6 +250,33 @@ public class TestSearch extends ServerBa
assertEquals(20, seenIDs.size());
}
+ public void testSearchWithTimeout() throws Exception {
+ deleteAllDocs();
+ LineFileDocs docs = new LineFileDocs(random());
+ long charCountLimit = atLeast(10000*1024);
+ long charCount = 0;
+ int id = 0;
+ while (charCount < charCountLimit) {
+ Document doc = docs.nextDoc();
+ charCount += doc.get("body").length() + doc.get("titleTokenized").length();
+ JSONObject fields = new JSONObject();
+ fields.put("title", doc.get("titleTokenized"));
+ fields.put("body", doc.get("body"));
+ fields.put("id", id++);
+ JSONObject o = new JSONObject();
+ o.put("fields", fields);
+ send("addDocument", o);
+ }
+
+ assertFailsWith("search",
+ "{queryText: the, retrieveFields: [id], timeoutSec: 0.0}",
+ "search > timeoutSec: must be > 0 msec");
+
+ // NOTE: does not actually test that we hit the timeout;
+ // just that we can specify timeoutSec
+ send("search", "{queryText: the, retrieveFields: [id], timeoutSec: 0.001}");
+ }
+
public void testSearchAfterWithSort() throws Exception {
deleteAllDocs();
long gen = 0;