You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/04/11 00:13:15 UTC
svn commit: r1466709 [2/2] - in /lucene/dev/trunk/lucene:
core/src/java/org/apache/lucene/search/ core/src/test/org/apache/lucene/
core/src/test/org/apache/lucene/codecs/perfield/
core/src/test/org/apache/lucene/index/ core/src/test/org/apache/lucene/s...
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Wed Apr 10 22:13:13 2013
@@ -868,7 +868,7 @@ public class TestFSTs extends LuceneTest
// turn writer into reader:
final IndexReader r = w.getReader();
- final IndexSearcher s = new IndexSearcher(r);
+ final IndexSearcher s = newSearcher(r);
w.close();
final List<String> allIDsList = new ArrayList<String>(allIDs);
@@ -997,7 +997,7 @@ public class TestFSTs extends LuceneTest
if (VERBOSE) {
System.out.println("TEST: got reader=" + r);
}
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
w.close();
final List<String> allTermsList = new ArrayList<String>(allTerms);
Modified: lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (original)
+++ lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java Wed Apr 10 22:13:13 2013
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
@@ -201,6 +202,23 @@ class TermsIncludingScoreQuery extends Q
}
@Override
+ public void score(Collector collector) throws IOException {
+ score(collector, NO_MORE_DOCS, nextDocOutOfOrder());
+ }
+
+ @Override
+ public boolean score(Collector collector, int max, int firstDocID)
+ throws IOException {
+ assert collector.acceptsDocsOutOfOrder();
+ collector.setScorer(this);
+ int doc;
+ for (doc = firstDocID; doc < max; doc = nextDocOutOfOrder()) {
+ collector.collect(doc);
+ }
+ return doc != NO_MORE_DOCS;
+ }
+
+ @Override
public float score() throws IOException {
return scores[ords[scoreUpto]];
}
@@ -214,8 +232,7 @@ class TermsIncludingScoreQuery extends Q
return docsEnum != null ? docsEnum.docID() : DocIdSetIterator.NO_MORE_DOCS;
}
- @Override
- public int nextDoc() throws IOException {
+ int nextDocOutOfOrder() throws IOException {
if (docsEnum != null) {
int docId = docsEnum.nextDoc();
if (docId == DocIdSetIterator.NO_MORE_DOCS) {
@@ -240,6 +257,11 @@ class TermsIncludingScoreQuery extends Q
}
@Override
+ public int nextDoc() throws IOException {
+ throw new UnsupportedOperationException("nextDoc() isn't supported because doc ids are emitted out of order");
+ }
+
+ @Override
public int advance(int target) throws IOException {
throw new UnsupportedOperationException("advance() isn't supported because doc ids are emitted out of order");
}
@@ -247,7 +269,7 @@ class TermsIncludingScoreQuery extends Q
private int advanceForExplainOnly(int target) throws IOException {
int docId;
do {
- docId = nextDoc();
+ docId = nextDocOutOfOrder();
if (docId < target) {
int tempDocId = docsEnum.advance(target);
if (tempDocId == target) {
@@ -286,7 +308,7 @@ class TermsIncludingScoreQuery extends Q
}
@Override
- public int nextDoc() throws IOException {
+ int nextDocOutOfOrder() throws IOException {
if (docsEnum != null) {
int docId;
do {
Modified: lucene/dev/trunk/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/trunk/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Wed Apr 10 22:13:13 2013
@@ -500,7 +500,7 @@ public class TestBlockJoin extends Lucen
final IndexSearcher s = newSearcher(r);
- final IndexSearcher joinS = newSearcher(joinR);
+ final IndexSearcher joinS = new IndexSearcher(joinR);
final Filter parentsFilter = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("isParent", "x"))));
@@ -1074,7 +1074,7 @@ public class TestBlockJoin extends Lucen
IndexReader r = w.getReader();
w.close();
- IndexSearcher s = newSearcher(r);
+ IndexSearcher s = new IndexSearcher(r);
// Create a filter that defines "parent" documents in the index - in this case resumes
Filter parentsFilter = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("docType", "resume"))));
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Wed Apr 10 22:13:13 2013
@@ -223,14 +223,25 @@ public class AssertingAtomicReader exten
}
static enum DocsEnumState { START, ITERATING, FINISHED };
- static class AssertingDocsEnum extends FilterDocsEnum {
+
+ public static class AssertingDocsEnum extends FilterDocsEnum {
private DocsEnumState state = DocsEnumState.START;
private int doc;
public AssertingDocsEnum(DocsEnum in) {
+ this(in, true);
+ }
+
+ public AssertingDocsEnum(DocsEnum in, boolean failOnUnsupportedDocID) {
super(in);
- int docid = in.docID();
- assert docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS : "invalid initial doc id: " + docid;
+ try {
+ int docid = in.docID();
+ assert docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS : in.getClass() + ": invalid initial doc id: " + docid;
+ } catch (UnsupportedOperationException e) {
+ if (failOnUnsupportedDocID) {
+ throw e;
+ }
+ }
doc = -1;
}
@@ -238,7 +249,7 @@ public class AssertingAtomicReader exten
public int nextDoc() throws IOException {
assert state != DocsEnumState.FINISHED : "nextDoc() called after NO_MORE_DOCS";
int nextDoc = super.nextDoc();
- assert nextDoc > doc : "backwards nextDoc from " + doc + " to " + nextDoc;
+ assert nextDoc > doc : "backwards nextDoc from " + doc + " to " + nextDoc + " " + in;
if (nextDoc == DocIdSetIterator.NO_MORE_DOCS) {
state = DocsEnumState.FINISHED;
} else {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java Wed Apr 10 22:13:13 2013
@@ -17,20 +17,28 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
import java.util.Random;
+import java.util.WeakHashMap;
import java.util.concurrent.ExecutorService;
-import java.io.IOException;
+import org.apache.lucene.index.AssertingAtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.VirtualMethod;
import org.apache.lucene.util._TestUtil;
-/**
+/**
* Helper class that adds some extra checks to ensure correct
* usage of {@code IndexSearcher} and {@code Weight}.
- * TODO: Extend this by more checks, that's just a start.
*/
public class AssertingIndexSearcher extends IndexSearcher {
final Random random;
@@ -58,16 +66,7 @@ public class AssertingIndexSearcher exte
@Override
public Weight createNormalizedWeight(Query query) throws IOException {
final Weight w = super.createNormalizedWeight(query);
- return new Weight() {
- @Override
- public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- return w.explain(context, doc);
- }
-
- @Override
- public Query getQuery() {
- return w.getQuery();
- }
+ return new AssertingWeight(random, w) {
@Override
public void normalize(float norm, float topLevelBoost) {
@@ -75,41 +74,271 @@ public class AssertingIndexSearcher exte
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
- boolean topScorer, Bits acceptDocs) throws IOException {
- Scorer scorer = w.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
- if (scorer != null) {
- // check that scorer obeys disi contract for docID() before next()/advance
- try {
- int docid = scorer.docID();
- assert docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS;
- } catch (UnsupportedOperationException ignored) {
- // from a top-level BS1
- assert topScorer;
- }
- }
- return scorer;
- }
-
- @Override
public float getValueForNormalization() {
throw new IllegalStateException("Weight already normalized.");
}
- @Override
- public boolean scoresDocsOutOfOrder() {
- // TODO: if this returns false, we should wrap
- // Scorer with AssertingScorer that confirms docIDs
- // are in order?
- return w.scoresDocsOutOfOrder();
- }
};
}
-
+
@Override
protected Query wrapFilter(Query query, Filter filter) {
if (random.nextBoolean())
return super.wrapFilter(query, filter);
return (filter == null) ? query : new FilteredQuery(query, filter, _TestUtil.randomFilterStrategy(random));
}
+
+ @Override
+ protected void search(List<AtomicReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+ super.search(leaves, AssertingWeight.wrap(random, weight), collector);
+ }
+
+ static class AssertingWeight extends Weight {
+
+ static Weight wrap(Random random, Weight other) {
+ return other instanceof AssertingWeight ? other : new AssertingWeight(random, other);
+ }
+
+ final Random random;
+ final Weight in;
+
+ AssertingWeight(Random random, Weight in) {
+ this.random = random;
+ this.in = in;
+ }
+
+ @Override
+ public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
+ return in.explain(context, doc);
+ }
+
+ @Override
+ public Query getQuery() {
+ return in.getQuery();
+ }
+
+ @Override
+ public float getValueForNormalization() throws IOException {
+ return in.getValueForNormalization();
+ }
+
+ @Override
+ public void normalize(float norm, float topLevelBoost) {
+ in.normalize(norm, topLevelBoost);
+ }
+
+ @Override
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ // if the caller asks for in-order scoring or if the weight does not support
+ // out-of order scoring then collection will have to happen in-order.
+ final boolean inOrder = scoreDocsInOrder || !scoresDocsOutOfOrder();
+ final Scorer inScorer = in.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
+ return AssertingScorer.wrap(new Random(random.nextLong()), inScorer, topScorer, inOrder);
+ }
+
+ @Override
+ public boolean scoresDocsOutOfOrder() {
+ return in.scoresDocsOutOfOrder();
+ }
+
+ }
+
+ enum TopScorer {
+ YES, NO, UNKNOWN;
+ }
+
+ public static class AssertingScorer extends Scorer {
+
+ private static final VirtualMethod<Scorer> SCORE_COLLECTOR = new VirtualMethod<Scorer>(Scorer.class, "score", Collector.class);
+ private static final VirtualMethod<Scorer> SCORE_COLLECTOR_RANGE = new VirtualMethod<Scorer>(Scorer.class, "score", Collector.class, int.class, int.class);
+
+ // we need to track scorers using a weak hash map because otherwise we
+ // could loose references because of eg.
+ // AssertingScorer.score(Collector) which needs to delegate to work correctly
+ private static Map<Scorer, WeakReference<AssertingScorer>> ASSERTING_INSTANCES = Collections.synchronizedMap(new WeakHashMap<Scorer, WeakReference<AssertingScorer>>());
+
+ private static Scorer wrap(Random random, Scorer other, TopScorer topScorer, boolean inOrder) {
+ if (other == null || other instanceof AssertingScorer) {
+ return other;
+ }
+ final AssertingScorer assertScorer = new AssertingScorer(random, other, topScorer, inOrder);
+ ASSERTING_INSTANCES.put(other, new WeakReference<AssertingScorer>(assertScorer));
+ return assertScorer;
+ }
+
+ static Scorer wrap(Random random, Scorer other, boolean topScorer, boolean inOrder) {
+ return wrap(random, other, topScorer ? TopScorer.YES : TopScorer.NO, inOrder);
+ }
+
+ static Scorer getAssertingScorer(Random random, Scorer other) {
+ if (other == null || other instanceof AssertingScorer) {
+ return other;
+ }
+ final WeakReference<AssertingScorer> assertingScorerRef = ASSERTING_INSTANCES.get(other);
+ final AssertingScorer assertingScorer = assertingScorerRef == null ? null : assertingScorerRef.get();
+ if (assertingScorer == null) {
+ // can happen in case of memory pressure or if
+ // scorer1.score(collector) calls
+ // collector.setScorer(scorer2) with scorer1 != scorer2, such as
+ // BooleanScorer. In that case we can't enable all assertions
+ return new AssertingScorer(random, other, TopScorer.UNKNOWN, false);
+ } else {
+ return assertingScorer;
+ }
+ }
+
+ final Random random;
+ final Scorer in;
+ final AssertingAtomicReader.AssertingDocsEnum docsEnumIn;
+ final TopScorer topScorer;
+ final boolean inOrder;
+ final boolean canCallNextDoc;
+
+ private AssertingScorer(Random random, Scorer in, TopScorer topScorer, boolean inOrder) {
+ super(in.weight);
+ this.random = random;
+ this.in = in;
+ this.topScorer = topScorer;
+ this.inOrder = inOrder;
+ this.docsEnumIn = new AssertingAtomicReader.AssertingDocsEnum(in, topScorer == TopScorer.NO);
+ this.canCallNextDoc = topScorer != TopScorer.YES // not a top scorer
+ || !SCORE_COLLECTOR_RANGE.isOverriddenAsOf(in.getClass()) // the default impl relies upon nextDoc()
+ || !SCORE_COLLECTOR.isOverriddenAsOf(in.getClass()); // the default impl relies upon nextDoc()
+ }
+
+ public Scorer getIn() {
+ return in;
+ }
+
+ boolean iterating() {
+ switch (docID()) {
+ case -1:
+ case NO_MORE_DOCS:
+ return false;
+ default:
+ return true;
+ }
+ }
+
+ @Override
+ public float score() throws IOException {
+ assert iterating();
+ final float score = in.score();
+ assert !Float.isNaN(score);
+ assert !Float.isNaN(score);
+ return score;
+ }
+
+ @Override
+ public void score(Collector collector) throws IOException {
+ assert topScorer != TopScorer.NO;
+ if (SCORE_COLLECTOR.isOverriddenAsOf(this.in.getClass())) {
+ if (random.nextBoolean()) {
+ try {
+ final boolean remaining = in.score(collector, DocsEnum.NO_MORE_DOCS, in.nextDoc());
+ assert !remaining;
+ } catch (UnsupportedOperationException e) {
+ in.score(collector);
+ }
+ } else {
+ in.score(collector);
+ }
+ } else {
+ // score(Collector) has not been overridden, use the super method in
+ // order to benefit from all assertions
+ super.score(collector);
+ }
+ }
+
+ @Override
+ public boolean score(Collector collector, int max, int firstDocID) throws IOException {
+ assert topScorer != TopScorer.NO;
+ if (SCORE_COLLECTOR_RANGE.isOverriddenAsOf(this.in.getClass())) {
+ return in.score(collector, max, firstDocID);
+ } else {
+ // score(Collector,int,int) has not been overridden, use the super
+ // method in order to benefit from all assertions
+ return super.score(collector, max, firstDocID);
+ }
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return in.getChildren();
+ }
+
+ @Override
+ public int freq() throws IOException {
+ assert iterating();
+ return in.freq();
+ }
+
+ @Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assert canCallNextDoc : "top scorers should not call nextDoc()";
+ return docsEnumIn.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ assert canCallNextDoc : "top scorers should not call advance(target)";
+ return docsEnumIn.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+ }
+
+ static class AssertingCollector extends Collector {
+
+ static Collector wrap(Random random, Collector other, boolean inOrder) {
+ return other instanceof AssertingCollector ? other : new AssertingCollector(random, other, inOrder);
+ }
+
+ final Random random;
+ final Collector in;
+ final boolean inOrder;
+ int lastCollected;
+
+ AssertingCollector(Random random, Collector in, boolean inOrder) {
+ this.random = random;
+ this.in = in;
+ this.inOrder = inOrder;
+ lastCollected = -1;
+ }
+
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ in.setScorer(AssertingScorer.getAssertingScorer(random, scorer));
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (inOrder || !acceptsDocsOutOfOrder()) {
+ assert doc > lastCollected : "Out of order : " + lastCollected + " " + doc;
+ }
+ in.collect(doc);
+ lastCollected = doc;
+ }
+
+ @Override
+ public void setNextReader(AtomicReaderContext context) throws IOException {
+ lastCollected = -1;
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return in.acceptsDocsOutOfOrder();
+ }
+
+ }
+
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1466709&r1=1466708&r2=1466709&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Apr 10 22:13:13 2013
@@ -1237,7 +1237,7 @@ public abstract class LuceneTestCase ext
* Create a new searcher over the reader. This searcher might randomly use
* threads.
*/
- public static IndexSearcher newSearcher(IndexReader r) throws IOException {
+ public static IndexSearcher newSearcher(IndexReader r) {
return newSearcher(r, true);
}
@@ -1246,18 +1246,26 @@ public abstract class LuceneTestCase ext
* threads. if <code>maybeWrap</code> is true, this searcher might wrap the
* reader with one that returns null for getSequentialSubReaders.
*/
- public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException {
+ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) {
Random random = random();
if (usually()) {
if (maybeWrap) {
- r = maybeWrapReader(r);
+ try {
+ r = maybeWrapReader(r);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
}
// TODO: this whole check is a coverage hack, we should move it to tests for various filterreaders.
// ultimately whatever you do will be checkIndex'd at the end anyway.
if (random.nextInt(500) == 0 && r instanceof AtomicReader) {
// TODO: not useful to check DirectoryReader (redundant with checkindex)
// but maybe sometimes run this on the other crazy readers maybeWrapReader creates?
- _TestUtil.checkReader(r);
+ try {
+ _TestUtil.checkReader(r);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
}
IndexSearcher ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r) : new AssertingIndexSearcher(random, r.getContext());
ret.setSimilarity(classEnvRule.similarity);