You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2015/08/03 14:39:16 UTC
svn commit: r1693892 - in /lucene/dev/trunk/lucene: ./
misc/src/java/org/apache/lucene/index/
misc/src/java/org/apache/lucene/search/
misc/src/test/org/apache/lucene/search/
Author: cpoerschke
Date: Mon Aug 3 12:39:16 2015
New Revision: 1693892
URL: http://svn.apache.org/r1693892
Log:
LUCENE-6691: SortingMergePolicy.(getSortDescription|isSorted) now considers FilterLeafReader instances. EarlyTerminatingSortingCollector.terminatedEarly accessor added. TestEarlyTerminatingSortingCollector.testTerminatedEarly test added.
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1693892&r1=1693891&r2=1693892&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Aug 3 12:39:16 2015
@@ -357,6 +357,11 @@ Changes in Runtime Behavior
when an unexpected, tragic exception strikes while merging. (Robert
Muir, Mike McCandless)
+* LUCENE-6691: SortingMergePolicy.isSorted now considers FilterLeafReader instances.
+ EarlyTerminatingSortingCollector.terminatedEarly accessor added.
+ TestEarlyTerminatingSortingCollector.testTerminatedEarly test added.
+ (Christine Poerschke)
+
Optimizations
* LUCENE-6548: Some optimizations for BlockTree's intersect with very
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java?rev=1693892&r1=1693891&r2=1693892&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java Mon Aug 3 12:39:16 2015
@@ -204,6 +204,8 @@ public final class SortingMergePolicy ex
if (diagnostics != null) {
return diagnostics.get(SORTER_ID_PROP);
}
+ } else if (reader instanceof FilterLeafReader) {
+ return getSortDescription(FilterLeafReader.unwrap(reader));
}
return null;
}
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java?rev=1693892&r1=1693891&r2=1693892&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/search/EarlyTerminatingSortingCollector.java Mon Aug 3 12:39:16 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexWriter;
@@ -86,6 +87,7 @@ public class EarlyTerminatingSortingColl
/** Number of documents to collect in each segment */
protected final int numDocsToCollect;
private final Sort mergePolicySort;
+ private final AtomicBoolean terminatedEarly = new AtomicBoolean(false);
/**
* Create a new {@link EarlyTerminatingSortingCollector} instance.
@@ -127,6 +129,7 @@ public class EarlyTerminatingSortingColl
public void collect(int doc) throws IOException {
super.collect(doc);
if (++numCollected >= numDocsToCollect) {
+ terminatedEarly.set(true);
throw new CollectionTerminatedException();
}
}
@@ -137,4 +140,8 @@ public class EarlyTerminatingSortingColl
}
}
+ public boolean terminatedEarly() {
+ return terminatedEarly.get();
+ }
+
}
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java?rev=1693892&r1=1693891&r2=1693892&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/search/TestEarlyTerminatingSortingCollector.java Mon Aug 3 12:39:16 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@@ -29,9 +30,12 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.QueryTimeout;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SortingMergePolicy;
@@ -47,6 +51,8 @@ import org.apache.lucene.search.SortFiel
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.uninverting.UninvertingReader;
+import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -61,6 +67,7 @@ public class TestEarlyTerminatingSorting
private RandomIndexWriter iw;
private IndexReader reader;
private SortingMergePolicy mergePolicy;
+ private final int forceMergeMaxSegmentCount = 5;
@Override
public void setUp() throws Exception {
@@ -76,7 +83,7 @@ public class TestEarlyTerminatingSorting
return doc;
}
- private void createRandomIndex() throws IOException {
+ private void createRandomIndex(Integer maxSegmentCount) throws IOException {
dir = newDirectory();
numDocs = atLeast(150);
final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5);
@@ -103,8 +110,11 @@ public class TestEarlyTerminatingSorting
iw.deleteDocuments(new Term("s", term));
}
}
- if (random().nextBoolean()) {
- iw.forceMerge(5);
+ if (maxSegmentCount != null) {
+ iw.forceMerge(maxSegmentCount.intValue());
+ }
+ else if (random().nextBoolean()) {
+ iw.forceMerge(forceMergeMaxSegmentCount);
}
reader = iw.getReader();
}
@@ -118,7 +128,7 @@ public class TestEarlyTerminatingSorting
public void testEarlyTermination() throws IOException {
final int iters = atLeast(8);
for (int i = 0; i < iters; ++i) {
- createRandomIndex();
+ createRandomIndex(null);
for (int j = 0; j < iters; ++j) {
final IndexSearcher searcher = newSearcher(reader);
final int numHits = TestUtil.nextInt(random(), 1, numDocs);
@@ -175,7 +185,7 @@ public class TestEarlyTerminatingSorting
}
public void testEarlyTerminationDifferentSorter() throws IOException {
- createRandomIndex();
+ createRandomIndex(null);
final int iters = atLeast(3);
for (int i = 0; i < iters; ++i) {
final IndexSearcher searcher = newSearcher(reader);
@@ -222,4 +232,69 @@ public class TestEarlyTerminatingSorting
}
}
+ private class TestTerminatedEarlySimpleCollector extends SimpleCollector {
+ private boolean collectedSomething;
+ public boolean collectedSomething() {
+ return collectedSomething;
+ }
+ @Override
+ public void collect(int doc) throws IOException {
+ collectedSomething = true;
+ }
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+ }
+
+ private class TestEarlyTerminatingSortingcollectorQueryTimeout implements QueryTimeout {
+ final private boolean shouldExit;
+ public TestEarlyTerminatingSortingcollectorQueryTimeout(boolean shouldExit) {
+ this.shouldExit = shouldExit;
+ }
+ public boolean shouldExit() {
+ return shouldExit;
+ }
+ }
+
+ private IndexSearcher newSearcherForTestTerminatedEarly(IndexReader r) throws IOException {
+ switch(random().nextInt(2)) {
+ case 0:
+ return new IndexSearcher(r);
+ case 1:
+ assertTrue(r+" is not a DirectoryReader", (r instanceof DirectoryReader));
+ final DirectoryReader directoryReader = ExitableDirectoryReader.wrap(
+ UninvertingReader.wrap((DirectoryReader) r, new HashMap<String,Type>()),
+ new TestEarlyTerminatingSortingcollectorQueryTimeout(false));
+ return new IndexSearcher(directoryReader);
+ }
+ fail("newSearcherForTestTerminatedEarly("+r+") fell through switch");
+ return null;
+ }
+
+ public void testTerminatedEarly() throws IOException {
+ final int iters = atLeast(8);
+ for (int i = 0; i < iters; ++i) {
+ createRandomIndex(1);
+
+ final IndexSearcher searcher = newSearcherForTestTerminatedEarly(reader); // future TODO: use newSearcher(reader);
+ final Query query = new MatchAllDocsQuery(); // search for everything/anything
+
+ final TestTerminatedEarlySimpleCollector collector1 = new TestTerminatedEarlySimpleCollector();
+ searcher.search(query, collector1);
+
+ final TestTerminatedEarlySimpleCollector collector2 = new TestTerminatedEarlySimpleCollector();
+ final EarlyTerminatingSortingCollector etsCollector = new EarlyTerminatingSortingCollector(collector2, sort, 1, mergePolicy.getSort());
+ searcher.search(query, etsCollector);
+
+ assertTrue("collector1="+collector1.collectedSomething()+" vs. collector2="+collector2.collectedSomething(), collector1.collectedSomething() == collector2.collectedSomething());
+
+ if (collector1.collectedSomething()) {
+ // we collected something and since we modestly asked for just one document we should have terminated early
+ assertTrue("should have terminated early (searcher.reader="+searcher.reader+")", etsCollector.terminatedEarly());
+ }
+ closeIndex();
+ }
+ }
+
}