You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2022/07/15 17:11:45 UTC
[lucene] branch branch_9x updated (f014c97aa26 -> 5cd6eda8cab)
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from f014c97aa26 LUCENE-10523: factor out UnifiedHighlighter.newFieldHighlighter() method (#821)
new aa082b46f66 LUCENE-10151: Adding Timeout Support to IndexSearcher (#927)
new 5cd6eda8cab CHANGES entry for LUCENE-10151
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 2 +
.../org/apache/lucene/search/IndexSearcher.java | 37 +++++++---
.../lucene/search/TimeLimitingBulkScorer.java | 75 +++++++++++++++++++
.../lucene/search/TestTimeLimitingBulkScorer.java | 84 ++++++++++++++++++++++
4 files changed, 190 insertions(+), 8 deletions(-)
create mode 100644 lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingBulkScorer.java
[lucene] 01/02: LUCENE-10151: Adding Timeout Support to IndexSearcher (#927)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit aa082b46f669f71cd0deb2e409c62be863f17091
Author: Deepika0510 <10...@users.noreply.github.com>
AuthorDate: Wed Jun 29 20:02:12 2022 +0530
LUCENE-10151: Adding Timeout Support to IndexSearcher (#927)
Authored-by: Deepika Sharma <dp...@amazon.com>
---
build.gradle | 2 +-
.../org/apache/lucene/search/IndexSearcher.java | 37 +++++++---
.../lucene/search/TimeLimitingBulkScorer.java | 75 +++++++++++++++++++
.../lucene/search/TestTimeLimitingBulkScorer.java | 84 ++++++++++++++++++++++
4 files changed, 189 insertions(+), 9 deletions(-)
diff --git a/build.gradle b/build.gradle
index 91164e73333..f2c2b565e47 100644
--- a/build.gradle
+++ b/build.gradle
@@ -182,4 +182,4 @@ apply from: file('gradle/hacks/hashmapAssertions.gradle')
apply from: file('gradle/hacks/turbocharge-jvm-opts.gradle')
apply from: file('gradle/hacks/dummy-outputs.gradle')
-apply from: file('gradle/pylucene/pylucene.gradle')
+apply from: file('gradle/pylucene/pylucene.gradle')
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index 0f6bdfdeb10..42d99d878d8 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.QueryTimeout;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
@@ -83,6 +84,8 @@ public class IndexSearcher {
static int maxClauseCount = 1024;
private static QueryCache DEFAULT_QUERY_CACHE;
private static QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
+ private QueryTimeout queryTimeout = null;
+ private boolean partialResult = false;
static {
final int maxCachedQueries = 1000;
@@ -484,6 +487,10 @@ public class IndexSearcher {
return search(query, manager);
}
+ public void setTimeout(QueryTimeout queryTimeout) throws IOException {
+ this.queryTimeout = queryTimeout;
+ }
+
/**
* Finds the top <code>n</code> hits for <code>query</code>.
*
@@ -507,6 +514,9 @@ public class IndexSearcher {
search(leafContexts, createWeight(query, results.scoreMode(), 1), results);
}
+ public boolean timedOut() {
+ return partialResult;
+ }
/**
* Search implementation with arbitrary sorting, plus control over whether hit scores and max
* score should be computed. Finds the top <code>n</code> hits for <code>query</code>, and sorting
@@ -720,18 +730,29 @@ public class IndexSearcher {
}
BulkScorer scorer = weight.bulkScorer(ctx);
if (scorer != null) {
- try {
- scorer.score(leafCollector, ctx.reader().getLiveDocs());
- } catch (
- @SuppressWarnings("unused")
- CollectionTerminatedException e) {
- // collection was terminated prematurely
- // continue with the following leaf
+ if (queryTimeout != null) {
+ TimeLimitingBulkScorer timeLimitingBulkScorer =
+ new TimeLimitingBulkScorer(scorer, queryTimeout);
+ try {
+ timeLimitingBulkScorer.score(leafCollector, ctx.reader().getLiveDocs());
+ } catch (
+ @SuppressWarnings("unused")
+ TimeLimitingBulkScorer.TimeExceededException e) {
+ partialResult = true;
+ }
+ } else {
+ try {
+ scorer.score(leafCollector, ctx.reader().getLiveDocs());
+ } catch (
+ @SuppressWarnings("unused")
+ CollectionTerminatedException e) {
+ // collection was terminated prematurely
+ // continue with the following leaf
+ }
}
}
}
}
-
/**
* Expert: called to re-write queries into primitive queries.
*
diff --git a/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
new file mode 100644
index 00000000000..4f17d40dc7c
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import org.apache.lucene.index.QueryTimeout;
+import org.apache.lucene.util.Bits;
+
+/**
+ * The {@link TimeLimitingBulkScorer} is used to timeout search requests that take longer than the
+ * maximum allowed search time limit. After this time is exceeded, the search thread is stopped by
+ * throwing a {@link TimeLimitingBulkScorer.TimeExceededException}.
+ *
+ * @see org.apache.lucene.index.ExitableDirectoryReader
+ */
+public class TimeLimitingBulkScorer extends BulkScorer {
+ // We score chunks of documents at a time so as to avoid the cost of checking the timeout for
+ // every document we score.
+ static final int INTERVAL = 100;
+ /** Thrown when elapsed search time exceeds allowed search time. */
+ @SuppressWarnings("serial")
+ static class TimeExceededException extends RuntimeException {
+
+ private TimeExceededException() {
+ super("TimeLimit Exceeded");
+ }
+ }
+
+ private BulkScorer in;
+ private QueryTimeout queryTimeout;
+ /**
+ * Create a TimeLimitingBulkScorer wrapper over another {@link BulkScorer} with a specified
+ * timeout.
+ *
+ * @param bulkScorer the wrapped {@link BulkScorer}
+ * @param queryTimeout max time allowed for collecting hits after which {@link
+ * TimeLimitingBulkScorer.TimeExceededException} is thrown
+ */
+ public TimeLimitingBulkScorer(BulkScorer bulkScorer, QueryTimeout queryTimeout) {
+ this.in = bulkScorer;
+ this.queryTimeout = queryTimeout;
+ }
+
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+ while (min < max) {
+ final int newMax = (int) Math.min((long) min + INTERVAL, max);
+ if (queryTimeout.shouldExit() == true) {
+ throw new TimeLimitingBulkScorer.TimeExceededException();
+ }
+ min = in.score(collector, acceptDocs, min, newMax); // in is the wrapped bulk scorer
+ }
+ return min;
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingBulkScorer.java
new file mode 100644
index 00000000000..aca89a8bb6f
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingBulkScorer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/** Tests the {@link TimeLimitingBulkScorer}. */
+@LuceneTestCase.SuppressSysoutChecks(
+ bugUrl = "http://test.is.timing.sensitive.so.it.prints.instead.of.failing")
+public class TestTimeLimitingBulkScorer extends LuceneTestCase {
+
+ public void testTimeLimitingBulkScorer() throws Exception {
+ Directory directory = newDirectory();
+ IndexWriter writer =
+ new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random())));
+ int n = 10000;
+ for (int i = 0; i < n; i++) {
+ Document d = new Document();
+ d.add(newTextField("default", "ones ", Field.Store.YES));
+ writer.addDocument(d);
+ }
+ writer.forceMerge(1);
+ writer.commit();
+ writer.close();
+
+ DirectoryReader directoryReader;
+ IndexSearcher searcher;
+ TopDocs top;
+ ScoreDoc[] hits = null;
+
+ Query query = new TermQuery(new Term("default", "ones"));
+ directoryReader = DirectoryReader.open(directory);
+ searcher = new IndexSearcher(directoryReader);
+ searcher.setTimeout(CountingQueryTimeout(10));
+ top = searcher.search(query, n);
+ hits = top.scoreDocs;
+ assertTrue(
+ "Partial result and is aborted is true",
+ hits.length > 0 && hits.length < n && searcher.timedOut());
+ directoryReader.close();
+ directory.close();
+ }
+
+ public static QueryTimeout CountingQueryTimeout(int timeallowed) {
+
+ return new QueryTimeout() {
+ public static int counter = 0;
+
+ @Override
+ public boolean shouldExit() {
+ counter++;
+ if (counter == timeallowed) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isTimeoutEnabled() {
+ return true;
+ }
+ };
+ }
+}
[lucene] 02/02: CHANGES entry for LUCENE-10151
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 5cd6eda8caba5a93eeaf60215885ec3171707449
Author: Michael Sokolov <so...@amazon.com>
AuthorDate: Wed Jun 29 10:35:09 2022 -0400
CHANGES entry for LUCENE-10151
---
build.gradle | 2 +-
lucene/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/build.gradle b/build.gradle
index f2c2b565e47..91164e73333 100644
--- a/build.gradle
+++ b/build.gradle
@@ -182,4 +182,4 @@ apply from: file('gradle/hacks/hashmapAssertions.gradle')
apply from: file('gradle/hacks/turbocharge-jvm-opts.gradle')
apply from: file('gradle/hacks/dummy-outputs.gradle')
-apply from: file('gradle/pylucene/pylucene.gradle')
\ No newline at end of file
+apply from: file('gradle/pylucene/pylucene.gradle')
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c8c043b848d..a85e38ca32b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -21,6 +21,8 @@ New Features
* LUCENE-10274: Added facetsets module for high dimensional (hyper-rectangle) faceting
(Shai Erera, Marc D'Mello, Greg Miller)
+* LUCENE-10151 Enable timeout support in IndexSearcher. (Deepika Sharma)
+
Improvements
---------------------