You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/13 13:17:06 UTC
svn commit: r1372366 [5/8] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ ...
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/Terms.java Mon Aug 13 11:16:57 2012
@@ -104,6 +104,15 @@ public abstract class Terms {
* measures, this measure does not take deleted documents
* into account. */
public abstract int getDocCount() throws IOException;
+
+ /** Returns true if documents in this field store offsets. */
+ public abstract boolean hasOffsets();
+
+ /** Returns true if documents in this field store positions. */
+ public abstract boolean hasPositions();
+
+ /** Returns true if documents in this field store payloads. */
+ public abstract boolean hasPayloads();
public final static Terms[] EMPTY_ARRAY = new Terms[0];
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Mon Aug 13 11:16:57 2012
@@ -213,7 +213,11 @@ public class BooleanQuery extends Query
}
public float coord(int overlap, int maxOverlap) {
- return similarity.coord(overlap, maxOverlap);
+ // LUCENE-4300: in most cases of maxOverlap=1, BQ rewrites itself away,
+ // so coord() is not applied. But when BQ cannot optimize itself away
+ // for a single clause (minNrShouldMatch, prohibited clauses, etc), its
+ // important not to apply coord(1,1) for consistency, it might not be 1.0F
+ return maxOverlap == 1 ? 1F : similarity.coord(overlap, maxOverlap);
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java Mon Aug 13 11:16:57 2012
@@ -41,16 +41,16 @@ class BooleanScorer2 extends Scorer {
private final List<Scorer> prohibitedScorers;
private class Coordinator {
- float[] coordFactors = null;
- int maxCoord = 0; // to be increased for each non prohibited scorer
- int nrMatchers; // to be increased by score() of match counting scorers.
-
- void init(boolean disableCoord) { // use after all scorers have been added.
+ final float coordFactors[];
+
+ Coordinator(int maxCoord, boolean disableCoord) {
coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1];
for (int i = 0; i < coordFactors.length; i++) {
coordFactors[i] = disableCoord ? 1.0f : ((BooleanWeight)weight).coord(i, maxCoord);
}
}
+
+ int nrMatchers; // to be increased by score() of match counting scorers.
}
private final Coordinator coordinator;
@@ -92,15 +92,13 @@ class BooleanScorer2 extends Scorer {
if (minNrShouldMatch < 0) {
throw new IllegalArgumentException("Minimum number of optional scorers should not be negative");
}
- coordinator = new Coordinator();
this.minNrShouldMatch = minNrShouldMatch;
- coordinator.maxCoord = maxCoord;
optionalScorers = optional;
requiredScorers = required;
prohibitedScorers = prohibited;
+ coordinator = new Coordinator(maxCoord, disableCoord);
- coordinator.init(disableCoord);
countingSumScorer = makeCountingSumScorer(disableCoord);
}
@@ -177,7 +175,7 @@ class BooleanScorer2 extends Scorer {
List<Scorer> requiredScorers) throws IOException {
// each scorer from the list counted as a single matcher
final int requiredNrMatchers = requiredScorers.size();
- return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) {
+ return new ConjunctionScorer(weight, requiredScorers) {
private int lastScoredDoc = -1;
// Save the score of lastScoredDoc, so that we don't compute it more than
// once in score().
@@ -201,8 +199,8 @@ class BooleanScorer2 extends Scorer {
}
private Scorer dualConjunctionSumScorer(boolean disableCoord,
- Scorer req1, Scorer req2) throws IOException { // non counting.
- return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), req1, req2);
+ Scorer req1, Scorer req2) throws IOException { // non counting.
+ return new ConjunctionScorer(weight, req1, req2);
// All scorers match, so defaultSimilarity always has 1 as
// the coordination factor.
// Therefore the sum of the scores of two scorers
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java Mon Aug 13 11:16:57 2012
@@ -27,17 +27,15 @@ import java.util.Comparator;
class ConjunctionScorer extends Scorer {
private final Scorer[] scorers;
- private final float coord;
private int lastDoc = -1;
- public ConjunctionScorer(Weight weight, float coord, Collection<Scorer> scorers) throws IOException {
- this(weight, coord, scorers.toArray(new Scorer[scorers.size()]));
+ public ConjunctionScorer(Weight weight, Collection<Scorer> scorers) throws IOException {
+ this(weight, scorers.toArray(new Scorer[scorers.size()]));
}
- public ConjunctionScorer(Weight weight, float coord, Scorer... scorers) throws IOException {
+ public ConjunctionScorer(Weight weight, Scorer... scorers) throws IOException {
super(weight);
this.scorers = scorers;
- this.coord = coord;
for (int i = 0; i < scorers.length; i++) {
if (scorers[i].nextDoc() == NO_MORE_DOCS) {
@@ -135,7 +133,7 @@ class ConjunctionScorer extends Scorer {
for (int i = 0; i < scorers.length; i++) {
sum += scorers[i].score();
}
- return sum * coord;
+ return sum;
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java Mon Aug 13 11:16:57 2012
@@ -548,12 +548,7 @@ class UnionDocsAndPositionsEnum extends
@Override
public BytesRef getPayload() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasPayload() {
- throw new UnsupportedOperationException();
+ return null;
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Mon Aug 13 11:16:57 2012
@@ -118,8 +118,8 @@ public class PayloadTermQuery extends Sp
}
protected void processPayload(Similarity similarity) throws IOException {
- final DocsAndPositionsEnum postings = termSpans.getPostings();
- if (postings.hasPayload()) {
+ if (termSpans.isPayloadAvailable()) {
+ final DocsAndPositionsEnum postings = termSpans.getPostings();
payload = postings.getPayload();
if (payload != null) {
payloadScore = function.currentScore(doc, term.field(),
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Mon Aug 13 11:16:57 2012
@@ -126,7 +126,7 @@ public class NearSpansUnordered extends
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}
@@ -256,7 +256,7 @@ public class NearSpansUnordered extends
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
SpansCell pointer = min();
while (pointer != null) {
if (pointer.isPayloadAvailable()) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Mon Aug 13 11:16:57 2012
@@ -157,7 +157,7 @@ public class SpanNotQuery extends SpanQu
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return includeSpans.isPayloadAvailable();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Mon Aug 13 11:16:57 2012
@@ -247,7 +247,7 @@ public class SpanOrQuery extends SpanQue
}
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
Spans top = top();
return top != null && top.isPayloadAvailable();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Mon Aug 13 11:16:57 2012
@@ -176,7 +176,7 @@ public abstract class SpanPositionCheckQ
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Mon Aug 13 11:16:57 2012
@@ -82,6 +82,6 @@ public abstract class Spans {
*
* @return true if there is a payload available at this position that can be loaded
*/
- public abstract boolean isPayloadAvailable();
+ public abstract boolean isPayloadAvailable() throws IOException;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Mon Aug 13 11:16:57 2012
@@ -36,6 +36,7 @@ public class TermSpans extends Spans {
protected int freq;
protected int count;
protected int position;
+ protected boolean readPayload;
public TermSpans(DocsAndPositionsEnum postings, Term term) {
this.postings = postings;
@@ -64,6 +65,7 @@ public class TermSpans extends Spans {
}
position = postings.nextPosition();
count++;
+ readPayload = false;
return true;
}
@@ -78,7 +80,7 @@ public class TermSpans extends Spans {
count = 0;
position = postings.nextPosition();
count++;
-
+ readPayload = false;
return true;
}
@@ -101,6 +103,7 @@ public class TermSpans extends Spans {
@Override
public Collection<byte[]> getPayload() throws IOException {
final BytesRef payload = postings.getPayload();
+ readPayload = true;
final byte[] bytes;
if (payload != null) {
bytes = new byte[payload.length];
@@ -113,8 +116,8 @@ public class TermSpans extends Spans {
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
- return postings.hasPayload();
+ public boolean isPayloadAvailable() throws IOException {
+ return readPayload == false && postings.getPayload() != null;
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java Mon Aug 13 11:16:57 2012
@@ -62,7 +62,21 @@ public abstract class HashFunction imple
return loader.availableServices();
}
-
+ /**
+ * Reloads the hash function list from the given {@link ClassLoader}.
+ * Changes to the function list are visible after the method ends, all
+ * iterators ({@link #availableHashFunctionNames()},...) stay consistent.
+ *
+ * <p><b>NOTE:</b> Only new functions are added, existing ones are
+ * never removed or replaced.
+ *
+ * <p><em>This method is expensive and should only be called for discovery
+ * of new functions on the given classpath/classloader!</em>
+ */
+ public static void reloadHashFunctions(ClassLoader classloader) {
+ loader.reload(classloader);
+ }
+
@Override
public String toString() {
return name;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java Mon Aug 13 11:16:57 2012
@@ -28,14 +28,18 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
/**
* Test indexes ~82M docs with 26 terms each, so you get > Integer.MAX_VALUE terms/docs pairs
* @lucene.experimental
*/
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
+@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
public class Test2BPostings extends LuceneTestCase {
@Nightly
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Mon Aug 13 11:16:57 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Iterator;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -258,9 +259,10 @@ public class TestCodecs extends LuceneTe
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR));
- final FieldsEnum fieldsEnum = reader.iterator();
- assertNotNull(fieldsEnum.next());
- final Terms terms2 = fieldsEnum.terms();
+ final Iterator<String> fieldsEnum = reader.iterator();
+ String fieldName = fieldsEnum.next();
+ assertNotNull(fieldName);
+ final Terms terms2 = reader.terms(fieldName);
assertNotNull(terms2);
final TermsEnum termsEnum = terms2.iterator(null);
@@ -286,7 +288,7 @@ public class TestCodecs extends LuceneTe
assertEquals(termsEnum.seekCeil(new BytesRef(terms[i].text2)), TermsEnum.SeekStatus.FOUND);
}
- assertNull(fieldsEnum.next());
+ assertFalse(fieldsEnum.hasNext());
reader.close();
dir.close();
}
@@ -436,14 +438,14 @@ public class TestCodecs extends LuceneTe
final int pos = posEnum.nextPosition();
assertEquals(positions[i].pos, pos);
if (positions[i].payload != null) {
- assertTrue(posEnum.hasPayload());
+ assertNotNull(posEnum.getPayload());
if (random().nextInt(3) < 2) {
// Verify the payload bytes
final BytesRef otherPayload = posEnum.getPayload();
assertTrue("expected=" + positions[i].payload.toString() + " got=" + otherPayload.toString(), positions[i].payload.equals(otherPayload));
}
} else {
- assertFalse(posEnum.hasPayload());
+ assertNull(posEnum.getPayload());
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Mon Aug 13 11:16:57 2012
@@ -612,20 +612,20 @@ public void testFilesOpenClose() throws
}
// check dictionary and posting lists
- FieldsEnum fenum1 = MultiFields.getFields(index1).iterator();
- FieldsEnum fenum2 = MultiFields.getFields(index1).iterator();
- String field1 = null;
+ Fields fields1 = MultiFields.getFields(index1);
+ Fields fields2 = MultiFields.getFields(index2);
+ Iterator<String> fenum2 = fields2.iterator();
Bits liveDocs = MultiFields.getLiveDocs(index1);
- while((field1=fenum1.next()) != null) {
+ for (String field1 : fields1) {
assertEquals("Different fields", field1, fenum2.next());
- Terms terms1 = fenum1.terms();
+ Terms terms1 = fields1.terms(field1);
if (terms1 == null) {
- assertNull(fenum2.terms());
+ assertNull(fields2.terms(field1));
continue;
}
TermsEnum enum1 = terms1.iterator(null);
- Terms terms2 = fenum2.terms();
+ Terms terms2 = fields2.terms(field1);
assertNotNull(terms2);
TermsEnum enum2 = terms2.iterator(null);
@@ -644,6 +644,7 @@ public void testFilesOpenClose() throws
}
}
}
+ assertFalse(fenum2.hasNext());
}
public void testGetIndexCommit() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Mon Aug 13 11:16:57 2012
@@ -32,7 +32,6 @@ import java.util.LinkedList;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
@@ -236,10 +235,9 @@ public class TestDoc extends LuceneTestC
for (int i = 0; i < reader.numDocs(); i++)
out.println(reader.document(i));
- FieldsEnum fis = reader.fields().iterator();
- String field = fis.next();
- while(field != null) {
- Terms terms = fis.terms();
+ Fields fields = reader.fields();
+ for (String field : fields) {
+ Terms terms = fields.terms(field);
assertNotNull(terms);
TermsEnum tis = terms.iterator(null);
while(tis.next() != null) {
@@ -259,7 +257,6 @@ public class TestDoc extends LuceneTestC
out.println("");
}
}
- field = fis.next();
}
reader.close();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java Mon Aug 13 11:16:57 2012
@@ -61,9 +61,7 @@ public class TestDocCount extends Lucene
if (fields == null) {
return;
}
- FieldsEnum e = fields.iterator();
- String field;
- while ((field = e.next()) != null) {
+ for (String field : fields) {
Terms terms = fields.terms(field);
if (terms == null) {
continue;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Mon Aug 13 11:16:57 2012
@@ -1025,4 +1025,21 @@ public class TestDocValuesIndexing exten
w.close();
d.close();
}
+
+ public void testFixedLengthNotReallyFixed() throws IOException {
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new DerefBytesDocValuesField("foo", new BytesRef("bar"), true));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new DerefBytesDocValuesField("foo", new BytesRef("bazz"), true));
+ try {
+ w.addDocument(doc);
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ w.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon Aug 13 11:16:57 2012
@@ -205,11 +205,11 @@ public class TestDocumentWriter extends
int freq = termPositions.freq();
assertEquals(3, freq);
assertEquals(0, termPositions.nextPosition());
- assertEquals(true, termPositions.hasPayload());
+ assertNotNull(termPositions.getPayload());
assertEquals(6, termPositions.nextPosition());
- assertEquals(false, termPositions.hasPayload());
+ assertNull(termPositions.getPayload());
assertEquals(7, termPositions.nextPosition());
- assertEquals(false, termPositions.hasPayload());
+ assertNull(termPositions.getPayload());
reader.close();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java Mon Aug 13 11:16:57 2012
@@ -27,12 +27,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeaks;
-
/**
* Tests for {@link DocumentsWriterStallControl}
*/
-@ThreadLeaks(failTestIfLeaking = true)
public class TestDocumentsWriterStallControl extends LuceneTestCase {
public void testSimpleStall() throws InterruptedException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Mon Aug 13 11:16:57 2012
@@ -173,15 +173,15 @@ public class TestDuelingCodecs extends L
}
assertFieldStatistics(leftFields, rightFields);
- FieldsEnum leftEnum = leftFields.iterator();
- FieldsEnum rightEnum = rightFields.iterator();
+ Iterator<String> leftEnum = leftFields.iterator();
+ Iterator<String> rightEnum = rightFields.iterator();
- String field;
- while ((field = leftEnum.next()) != null) {
+ while (leftEnum.hasNext()) {
+ String field = leftEnum.next();
assertEquals(info, field, rightEnum.next());
- assertTerms(leftEnum.terms(), rightEnum.terms(), deep);
+ assertTerms(leftFields.terms(field), rightFields.terms(field), deep);
}
- assertNull(rightEnum.next());
+ assertFalse(rightEnum.hasNext());
}
/**
@@ -207,6 +207,9 @@ public class TestDuelingCodecs extends L
return;
}
assertTermsStatistics(leftTerms, rightTerms);
+ assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets());
+ assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions());
+ assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads());
TermsEnum leftTermsEnum = leftTerms.iterator(null);
TermsEnum rightTermsEnum = rightTerms.iterator(null);
@@ -409,12 +412,9 @@ public class TestDuelingCodecs extends L
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
+ assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
- if (leftDocs.hasPayload()) {
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- }
}
}
assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
@@ -506,10 +506,7 @@ public class TestDuelingCodecs extends L
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
- if (leftDocs.hasPayload()) {
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- }
+ assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
}
}
@@ -528,9 +525,7 @@ public class TestDuelingCodecs extends L
return;
}
- FieldsEnum fieldsEnum = leftFields.iterator();
- String field;
- while ((field = fieldsEnum.next()) != null) {
+ for (String field : leftFields) {
DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
if (leftNorms != null && rightNorms != null) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java Mon Aug 13 11:16:57 2012
@@ -40,10 +40,7 @@ public class TestFilterAtomicReader exte
TestFields(Fields in) {
super(in);
}
- @Override
- public FieldsEnum iterator() throws IOException {
- return new TestFieldsEnum(super.iterator());
- }
+
@Override
public Terms terms(String field) throws IOException {
return new TestTerms(super.terms(field));
@@ -61,17 +58,6 @@ public class TestFilterAtomicReader exte
}
}
- private static class TestFieldsEnum extends FilterFieldsEnum {
- TestFieldsEnum(FieldsEnum in) {
- super(in);
- }
-
- @Override
- public Terms terms() throws IOException {
- return new TestTerms(super.terms());
- }
- }
-
private static class TestTermsEnum extends FilterTermsEnum {
public TestTermsEnum(TermsEnum in) {
super(in);
@@ -194,7 +180,6 @@ public class TestFilterAtomicReader exte
checkOverrideMethods(FilterAtomicReader.class);
checkOverrideMethods(FilterAtomicReader.FilterFields.class);
checkOverrideMethods(FilterAtomicReader.FilterTerms.class);
- checkOverrideMethods(FilterAtomicReader.FilterFieldsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterTermsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterDocsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterDocsAndPositionsEnum.class);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Aug 13 11:16:57 2012
@@ -526,8 +526,6 @@ public class TestIndexWriterOnDiskFull e
dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
final Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
try {
writer.addDocument(doc);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Mon Aug 13 11:16:57 2012
@@ -60,17 +60,22 @@ public class TestIndexableField extends
@Override
public boolean storeTermVectors() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return indexed() && counter % 2 == 1 && counter % 10 != 9;
}
@Override
public boolean storeTermVectorOffsets() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return storeTermVectors() && counter % 10 != 9;
}
@Override
public boolean storeTermVectorPositions() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return storeTermVectors() && counter % 10 != 9;
+ }
+
+ @Override
+ public boolean storeTermVectorPayloads() {
+ return storeTermVectors() && counter % 10 != 9;
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Mon Aug 13 11:16:57 2012
@@ -128,6 +128,7 @@ public class TestLazyProxSkipping extend
// check if the number of calls of seek() does not exceed the number of hits
assertTrue(this.seeksCounter > 0);
assertTrue("seeksCounter=" + this.seeksCounter + " numHits=" + numHits, this.seeksCounter <= numHits + 1);
+ searcher.getIndexReader().close();
}
public void testLazySkipping() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Mon Aug 13 11:16:57 2012
@@ -205,8 +205,11 @@ public class TestLongPostings extends Lu
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
- if (random().nextBoolean() && postings.hasPayload()) {
+ if (random().nextBoolean()) {
postings.getPayload();
+ if (random().nextBoolean()) {
+ postings.getPayload(); // get it again
+ }
}
}
}
@@ -247,8 +250,11 @@ public class TestLongPostings extends Lu
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
- if (random().nextBoolean() && postings.hasPayload()) {
+ if (random().nextBoolean()) {
postings.getPayload();
+ if (random().nextBoolean()) {
+ postings.getPayload(); // get it again
+ }
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java Mon Aug 13 11:16:57 2012
@@ -169,4 +169,22 @@ public class TestMultiFields extends Luc
r.close();
dir.close();
}
+
+ public void testTermDocsEnum() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document d = new Document();
+ d.add(newStringField("f", "j", Field.Store.NO));
+ w.addDocument(d);
+ w.commit();
+ w.addDocument(d);
+ IndexReader r = w.getReader();
+ w.close();
+ DocsEnum de = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j"));
+ assertEquals(0, de.nextDoc());
+ assertEquals(1, de.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java Mon Aug 13 11:16:57 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Iterator;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -94,20 +95,21 @@ public class TestParallelTermEnum extend
Bits liveDocs = pr.getLiveDocs();
- FieldsEnum fe = pr.fields().iterator();
+ Fields fields = pr.fields();
+ Iterator<String> fe = fields.iterator();
String f = fe.next();
assertEquals("field1", f);
- checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the");
+ checkTerms(fields.terms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
f = fe.next();
assertEquals("field2", f);
- checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the");
+ checkTerms(fields.terms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
f = fe.next();
assertEquals("field3", f);
- checkTerms(fe.terms(), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the");
+ checkTerms(fields.terms(f), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the");
- assertNull(fe.next());
+ assertFalse(fe.hasNext());
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Mon Aug 13 11:16:57 2012
@@ -145,8 +145,18 @@ public class TestPayloadProcessorProvide
Document doc = new Document();
doc.add(newField("id", "doc" + i, customType));
doc.add(newTextField("content", "doc content " + i, Field.Store.NO));
- doc.add(new TextField("p", payloadTS1));
- doc.add(new TextField("p", payloadTS2));
+ if (random.nextBoolean()) {
+ doc.add(new TextField("p", payloadTS1));
+ doc.add(new TextField("p", payloadTS2));
+ } else {
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setStoreTermVectors(true);
+ type.setStoreTermVectorPositions(true);
+ type.setStoreTermVectorPayloads(true);
+ type.setStoreTermVectorOffsets(random.nextBoolean());
+ doc.add(new Field("p", payloadTS1, type));
+ doc.add(new Field("p", payloadTS2, type));
+ }
writer.addDocument(doc);
if (multipleCommits && (i % 4 == 0)) {
writer.commit();
@@ -163,8 +173,8 @@ public class TestPayloadProcessorProvide
DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
tpe.nextPosition();
- if (tpe.hasPayload()) {
- BytesRef payload = tpe.getPayload();
+ BytesRef payload = tpe.getPayload();
+ if (payload != null) {
assertEquals(1, payload.length);
assertEquals(1, payload.bytes[0]);
++numPayloads;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java Mon Aug 13 11:16:57 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
+import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
@@ -200,18 +201,10 @@ public class TestPayloads extends Lucene
for (int i = 0; i < freq; i++) {
for (int j = 0; j < numTerms; j++) {
tps[j].nextPosition();
- if (tps[j].hasPayload()) {
- BytesRef br = tps[j].getPayload();
+ BytesRef br = tps[j].getPayload();
+ if (br != null) {
System.arraycopy(br.bytes, br.offset, verifyPayloadData, offset, br.length);
offset += br.length;
- // Just to ensure all codecs can
- // handle a caller that mucks with the
- // returned payload:
- if (rarely()) {
- br.bytes = new byte[random().nextInt(5)];
- }
- br.length = 0;
- br.offset = 0;
}
}
}
@@ -267,11 +260,6 @@ public class TestPayloads extends Lucene
tp.nextPosition();
assertEquals("Wrong payload length.", 3 * skipInterval - 2 * numDocs - 1, tp.getPayload().length);
- /*
- * Test multiple call of getPayload()
- */
- assertFalse(tp.hasPayload());
-
reader.close();
// test long payload
@@ -591,4 +579,73 @@ public class TestPayloads extends Lucene
dir.close();
}
+
+ /** some docs have payload att, some not */
+ public void testMixupDocs() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ Field field = new TextField("field", "", Field.Store.NO);
+ TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ doc.add(field);
+ writer.addDocument(doc);
+ Token withPayload = new Token("withPayload", 0, 11);
+ withPayload.setPayload(new BytesRef("test"));
+ ts = new CannedTokenStream(withPayload);
+ assertTrue(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ writer.addDocument(doc);
+ ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ writer.addDocument(doc);
+ DirectoryReader reader = writer.getReader();
+ AtomicReader sr = reader.getSequentialSubReaders().get(0);
+ DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
+ de.nextDoc();
+ de.nextPosition();
+ assertEquals(new BytesRef("test"), de.getPayload());
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
+ /** some field instances have payload att, some not */
+ public void testMixupMultiValued() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ Field field = new TextField("field", "", Field.Store.NO);
+ TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ doc.add(field);
+ Field field2 = new TextField("field", "", Field.Store.NO);
+ Token withPayload = new Token("withPayload", 0, 11);
+ withPayload.setPayload(new BytesRef("test"));
+ ts = new CannedTokenStream(withPayload);
+ assertTrue(ts.hasAttribute(PayloadAttribute.class));
+ field2.setTokenStream(ts);
+ doc.add(field2);
+ Field field3 = new TextField("field", "", Field.Store.NO);
+ ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field3.setTokenStream(ts);
+ doc.add(field3);
+ writer.addDocument(doc);
+ DirectoryReader reader = writer.getReader();
+ SegmentReader sr = getOnlySegmentReader(reader);
+ DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
+ de.nextDoc();
+ de.nextPosition();
+ assertEquals(new BytesRef("test"), de.getPayload());
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Mon Aug 13 11:16:57 2012
@@ -23,8 +23,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeMap;
@@ -731,20 +733,21 @@ public class TestPostingsFormat extends
System.out.println(" now check payload length=" + (position.payload == null ? 0 : position.payload.length));
}
if (position.payload == null || position.payload.length == 0) {
- assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
+ assertNull("should not have payload", docsAndPositionsEnum.getPayload());
} else {
- assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
-
BytesRef payload = docsAndPositionsEnum.getPayload();
- assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
+ assertNotNull("should have payload but doesn't", payload);
- assertNotNull("payload should not be null", payload);
assertEquals("payload length is wrong", position.payload.length, payload.length);
for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
assertEquals("payload bytes are wrong",
position.payload[byteUpto],
payload.bytes[payload.offset+byteUpto]);
}
+
+ // make a deep copy
+ payload = BytesRef.deepCopyOf(payload);
+ assertEquals("2nd call to getPayload returns something different!", payload, docsAndPositionsEnum.getPayload());
}
} else {
if (VERBOSE) {
@@ -888,6 +891,26 @@ public class TestPostingsFormat extends
}
}
}
+
+ private void testFields(Fields fields) throws Exception {
+ Iterator<String> iterator = fields.iterator();
+ while (iterator.hasNext()) {
+ String field = iterator.next();
+ try {
+ iterator.remove();
+ fail("Fields.iterator() allows for removal");
+ } catch (UnsupportedOperationException expected) {
+ // expected;
+ }
+ }
+ assertFalse(iterator.hasNext());
+ try {
+ iterator.next();
+ fail("Fields.iterator() doesn't throw NoSuchElementException when past the end");
+ } catch (NoSuchElementException expected) {
+ // expected
+ }
+ }
public void test() throws Exception {
Directory dir = newFSDirectory(_TestUtil.getTempDir("testPostingsFormat"));
@@ -897,6 +920,7 @@ public class TestPostingsFormat extends
FieldsProducer fieldsProducer = buildIndex(dir, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads);
+ testFields(fieldsProducer);
//testTerms(fieldsProducer, EnumSet.noneOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.LIVE_DOCS), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.LIVE_DOCS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Mon Aug 13 11:16:57 2012
@@ -170,7 +170,7 @@ public class TestPostingsOffsets extends
assertTrue(storedNumbers.substring(start, end).equals(term));
if (withPayloads) {
// check that we have a payload and it starts with "pos"
- assertTrue(dp.hasPayload());
+ assertNotNull(dp.getPayload());
BytesRef payload = dp.getPayload();
assertTrue(payload.utf8ToString().startsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
@@ -198,7 +198,7 @@ public class TestPostingsOffsets extends
assertTrue(storedNumbers.substring(start, end).equals("hundred"));
if (withPayloads) {
// check that we have a payload and it starts with "pos"
- assertTrue(dp.hasPayload());
+ assertNotNull(dp.getPayload());
BytesRef payload = dp.getPayload();
assertTrue(payload.utf8ToString().startsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java Mon Aug 13 11:16:57 2012
@@ -25,6 +25,7 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.MergedIterator;
import org.apache.lucene.util._TestUtil;
public class TestPrefixCodedTerms extends LuceneTestCase {
@@ -67,17 +68,20 @@ public class TestPrefixCodedTerms extend
assertFalse(expected.hasNext());
}
+ @SuppressWarnings("unchecked")
public void testMergeEmpty() {
List<Iterator<Term>> subs = Collections.emptyList();
- assertFalse(CoalescedDeletes.mergedIterator(subs).hasNext());
+ Iterator<Term> merged = new MergedIterator<Term>(subs.toArray(new Iterator[0]));
+ assertFalse(merged.hasNext());
subs = new ArrayList<Iterator<Term>>();
subs.add(new PrefixCodedTerms.Builder().finish().iterator());
subs.add(new PrefixCodedTerms.Builder().finish().iterator());
- Iterator<Term> merged = CoalescedDeletes.mergedIterator(subs);
+ merged = new MergedIterator<Term>(subs.toArray(new Iterator[0]));
assertFalse(merged.hasNext());
}
+ @SuppressWarnings("unchecked")
public void testMergeOne() {
Term t1 = new Term("foo", "a");
PrefixCodedTerms.Builder b1 = new PrefixCodedTerms.Builder();
@@ -93,13 +97,14 @@ public class TestPrefixCodedTerms extend
subs.add(pb1.iterator());
subs.add(pb2.iterator());
- Iterator<Term> merged = CoalescedDeletes.mergedIterator(subs);
+ Iterator<Term> merged = new MergedIterator<Term>(subs.toArray(new Iterator[0]));
assertTrue(merged.hasNext());
assertEquals(t1, merged.next());
assertTrue(merged.hasNext());
assertEquals(t2, merged.next());
}
+ @SuppressWarnings("unchecked")
public void testMergeRandom() {
PrefixCodedTerms pb[] = new PrefixCodedTerms[_TestUtil.nextInt(random(), 2, 10)];
Set<Term> superSet = new TreeSet<Term>();
@@ -126,7 +131,7 @@ public class TestPrefixCodedTerms extend
}
Iterator<Term> expected = superSet.iterator();
- Iterator<Term> actual = CoalescedDeletes.mergedIterator(subs);
+ Iterator<Term> actual = new MergedIterator<Term>(subs.toArray(new Iterator[0]));
while (actual.hasNext()) {
assertTrue(expected.hasNext());
assertEquals(expected.next(), actual.next());
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java Mon Aug 13 11:16:57 2012
@@ -67,13 +67,15 @@ public class TestReaderClosed extends Lu
// expected
}
}
-
+
// LUCENE-3800
public void testReaderChaining() throws Exception {
assertTrue(reader.getRefCount() > 0);
IndexReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader);
wrappedReader = new ParallelAtomicReader((AtomicReader) wrappedReader);
+
IndexSearcher searcher = newSearcher(wrappedReader);
+
TermRangeQuery query = TermRangeQuery.newStringRange("field", "a", "z", true, true);
searcher.search(query, 5);
reader.close(); // close original child reader
@@ -84,6 +86,9 @@ public class TestReaderClosed extends Lu
"this IndexReader cannot be used anymore as one of its child readers was closed",
ace.getMessage()
);
+ } finally {
+ // shutdown executor: in case of wrap-wrap-wrapping
+ searcher.getIndexReader().close();
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java Mon Aug 13 11:16:57 2012
@@ -114,10 +114,9 @@ public class TestSegmentReader extends L
}
public void testTerms() throws IOException {
- FieldsEnum fields = MultiFields.getFields(reader).iterator();
- String field;
- while((field = fields.next()) != null) {
- Terms terms = fields.terms();
+ Fields fields = MultiFields.getFields(reader);
+ for (String field : fields) {
+ Terms terms = fields.terms(field);
assertNotNull(terms);
TermsEnum termsEnum = terms.iterator(null);
while(termsEnum.next() != null) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Mon Aug 13 11:16:57 2012
@@ -396,13 +396,11 @@ public class TestStressIndexing2 extends
Fields tv1 = r1.getTermVectors(id1);
System.out.println(" d1=" + tv1);
if (tv1 != null) {
- FieldsEnum fieldsEnum = tv1.iterator();
- String field;
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
- while ((field=fieldsEnum.next()) != null) {
+ for (String field : tv1) {
System.out.println(" " + field + ":");
- Terms terms3 = fieldsEnum.terms();
+ Terms terms3 = tv1.terms(field);
assertNotNull(terms3);
TermsEnum termsEnum3 = terms3.iterator(null);
BytesRef term2;
@@ -430,13 +428,11 @@ public class TestStressIndexing2 extends
Fields tv2 = r2.getTermVectors(id2);
System.out.println(" d2=" + tv2);
if (tv2 != null) {
- FieldsEnum fieldsEnum = tv2.iterator();
- String field;
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
- while ((field=fieldsEnum.next()) != null) {
+ for (String field : tv2) {
System.out.println(" " + field + ":");
- Terms terms3 = fieldsEnum.terms();
+ Terms terms3 = tv2.terms(field);
assertNotNull(terms3);
TermsEnum termsEnum3 = terms3.iterator(null);
BytesRef term2;
@@ -469,8 +465,11 @@ public class TestStressIndexing2 extends
// Verify postings
//System.out.println("TEST: create te1");
- final FieldsEnum fields1 = MultiFields.getFields(r1).iterator();
- final FieldsEnum fields2 = MultiFields.getFields(r2).iterator();
+ final Fields fields1 = MultiFields.getFields(r1);
+ final Iterator<String> fields1Enum = fields1.iterator();
+ final Fields fields2 = MultiFields.getFields(r2);
+ final Iterator<String> fields2Enum = fields2.iterator();
+
String field1=null, field2=null;
TermsEnum termsEnum1 = null;
@@ -489,16 +488,15 @@ public class TestStressIndexing2 extends
for(;;) {
len1=0;
if (termsEnum1 == null) {
- field1 = fields1.next();
- if (field1 == null) {
+ if (!fields1Enum.hasNext()) {
break;
- } else {
- Terms terms = fields1.terms();
- if (terms == null) {
- continue;
- }
- termsEnum1 = terms.iterator(null);
}
+ field1 = fields1Enum.next();
+ Terms terms = fields1.terms(field1);
+ if (terms == null) {
+ continue;
+ }
+ termsEnum1 = terms.iterator(null);
}
term1 = termsEnum1.next();
if (term1 == null) {
@@ -523,16 +521,15 @@ public class TestStressIndexing2 extends
for(;;) {
len2=0;
if (termsEnum2 == null) {
- field2 = fields2.next();
- if (field2 == null) {
+ if (!fields2Enum.hasNext()) {
break;
- } else {
- Terms terms = fields2.terms();
- if (terms == null) {
- continue;
- }
- termsEnum2 = terms.iterator(null);
}
+ field2 = fields2Enum.next();
+ Terms terms = fields2.terms(field2);
+ if (terms == null) {
+ continue;
+ }
+ termsEnum2 = terms.iterator(null);
}
term2 = termsEnum2.next();
if (term2 == null) {
@@ -605,18 +602,17 @@ public class TestStressIndexing2 extends
}
assertTrue(d2 != null);
- FieldsEnum fieldsEnum1 = d1.iterator();
- FieldsEnum fieldsEnum2 = d2.iterator();
- String field1;
- while ((field1 = fieldsEnum1.next()) != null) {
+ Iterator<String> fieldsEnum2 = d2.iterator();
+
+ for (String field1 : d1) {
String field2 = fieldsEnum2.next();
assertEquals(field1, field2);
- Terms terms1 = fieldsEnum1.terms();
+ Terms terms1 = d1.terms(field1);
assertNotNull(terms1);
TermsEnum termsEnum1 = terms1.iterator(null);
- Terms terms2 = fieldsEnum2.terms();
+ Terms terms2 = d2.terms(field2);
assertNotNull(terms2);
TermsEnum termsEnum2 = terms2.iterator(null);
@@ -689,7 +685,7 @@ public class TestStressIndexing2 extends
assertNull(termsEnum2.next());
}
- assertNull(fieldsEnum2.next());
+ assertFalse(fieldsEnum2.hasNext());
}
private class IndexingThread extends Thread {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java Mon Aug 13 11:16:57 2012
@@ -74,9 +74,8 @@ public class TestSumDocFreq extends Luce
private void assertSumDocFreq(IndexReader ir) throws Exception {
// compute sumDocFreq across all fields
Fields fields = MultiFields.getFields(ir);
- FieldsEnum fieldEnum = fields.iterator();
- String f = null;
- while ((f = fieldEnum.next()) != null) {
+
+ for (String f : fields) {
Terms terms = fields.terms(f);
long sumDocFreq = terms.getSumDocFreq();
if (sumDocFreq == -1) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java Mon Aug 13 11:16:57 2012
@@ -122,6 +122,21 @@ public final class FieldCacheRewriteMeth
public long size() {
return -1;
}
+
+ @Override
+ public boolean hasOffsets() {
+ return false;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return false;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
});
assert termsEnum != null;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java Mon Aug 13 11:16:57 2012
@@ -65,7 +65,9 @@ public class TestBoolean2 extends Lucene
}
writer.close();
littleReader = DirectoryReader.open(directory);
- searcher = new IndexSearcher(littleReader);
+ searcher = newSearcher(littleReader);
+ // this is intentionally using the baseline sim, because it compares against bigSearcher (which uses a random one)
+ searcher.setSimilarity(new DefaultSimilarity());
// Make big index
dir2 = new MockDirectoryWrapper(random(), new RAMDirectory(directory, IOContext.DEFAULT));
@@ -261,7 +263,7 @@ public class TestBoolean2 extends Lucene
try {
// increase number of iterations for more complete testing
- int num = atLeast(10);
+ int num = atLeast(20);
for (int i=0; i<num; i++) {
int level = random().nextInt(3);
q1 = randBoolQuery(new Random(random().nextLong()), random().nextBoolean(), level, field, vals, null);
@@ -270,7 +272,14 @@ public class TestBoolean2 extends Lucene
// match up.
Sort sort = Sort.INDEXORDER;
- QueryUtils.check(random(), q1,searcher);
+ QueryUtils.check(random(), q1,searcher); // baseline sim
+ try {
+ // a little hackish, QueryUtils.check is too costly to do on bigSearcher in this loop.
+ searcher.setSimilarity(bigSearcher.getSimilarity()); // random sim
+ QueryUtils.check(random(), q1, searcher);
+ } finally {
+ searcher.setSimilarity(new DefaultSimilarity()); // restore
+ }
TopFieldCollector collector = TopFieldCollector.create(sort, 1000,
false, true, true, true);
@@ -321,6 +330,14 @@ public class TestBoolean2 extends Lucene
Query q;
if (qType < 3) {
q = new TermQuery(new Term(field, vals[rnd.nextInt(vals.length)]));
+ } else if (qType < 4) {
+ Term t1 = new Term(field, vals[rnd.nextInt(vals.length)]);
+ Term t2 = new Term(field, vals[rnd.nextInt(vals.length)]);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(t1);
+ pq.add(t2);
+ pq.setSlop(10); // increase possibility of matching
+ q = pq;
} else if (qType < 7) {
q = new WildcardQuery(new Term(field, "w*"));
} else {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java Mon Aug 13 11:16:57 2012
@@ -23,6 +23,8 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -297,8 +299,8 @@ public class TestBooleanMinShouldMatch e
}
public void testRandomQueries() throws Exception {
- String field="data";
- String[] vals = {"1","2","3","4","5","6","A","Z","B","Y","Z","X","foo"};
+ final String field="data";
+ final String[] vals = {"1","2","3","4","5","6","A","Z","B","Y","Z","X","foo"};
int maxLev=4;
// callback object to set a random setMinimumNumberShouldMatch
@@ -310,13 +312,18 @@ public class TestBooleanMinShouldMatch e
if (c[i].getOccur() == BooleanClause.Occur.SHOULD) opt++;
}
q.setMinimumNumberShouldMatch(random().nextInt(opt+2));
+ if (random().nextBoolean()) {
+ // also add a random negation
+ Term randomTerm = new Term(field, vals[random().nextInt(vals.length)]);
+ q.add(new TermQuery(randomTerm), BooleanClause.Occur.MUST_NOT);
+ }
}
};
// increase number of iterations for more complete testing
- int num = atLeast(10);
+ int num = atLeast(20);
for (int i=0; i<num; i++) {
int lev = random().nextInt(maxLev);
final long seed = random().nextLong();
@@ -336,44 +343,90 @@ public class TestBooleanMinShouldMatch e
QueryUtils.check(random(), q1,s);
QueryUtils.check(random(), q2,s);
}
- // The constrained query
- // should be a superset to the unconstrained query.
- if (top2.totalHits > top1.totalHits) {
- fail("Constrained results not a subset:\n"
- + CheckHits.topdocsString(top1,0,0)
- + CheckHits.topdocsString(top2,0,0)
- + "for query:" + q2.toString());
- }
-
- for (int hit=0; hit<top2.totalHits; hit++) {
- int id = top2.scoreDocs[hit].doc;
- float score = top2.scoreDocs[hit].score;
- boolean found=false;
- // find this doc in other hits
- for (int other=0; other<top1.totalHits; other++) {
- if (top1.scoreDocs[other].doc == id) {
- found=true;
- float otherScore = top1.scoreDocs[other].score;
- // check if scores match
- assertEquals("Doc " + id + " scores don't match\n"
- + CheckHits.topdocsString(top1,0,0)
- + CheckHits.topdocsString(top2,0,0)
- + "for query:" + q2.toString(),
- score, otherScore, CheckHits.explainToleranceDelta(score, otherScore));
- }
- }
+ assertSubsetOfSameScores(q2, top1, top2);
+ }
+ // System.out.println("Total hits:"+tot);
+ }
+
+ private void assertSubsetOfSameScores(Query q, TopDocs top1, TopDocs top2) {
+ // The constrained query
+ // should be a subset to the unconstrained query.
+ if (top2.totalHits > top1.totalHits) {
+ fail("Constrained results not a subset:\n"
+ + CheckHits.topdocsString(top1,0,0)
+ + CheckHits.topdocsString(top2,0,0)
+ + "for query:" + q.toString());
+ }
- // check if subset
- if (!found) fail("Doc " + id + " not found\n"
+ for (int hit=0; hit<top2.totalHits; hit++) {
+ int id = top2.scoreDocs[hit].doc;
+ float score = top2.scoreDocs[hit].score;
+ boolean found=false;
+ // find this doc in other hits
+ for (int other=0; other<top1.totalHits; other++) {
+ if (top1.scoreDocs[other].doc == id) {
+ found=true;
+ float otherScore = top1.scoreDocs[other].score;
+ // check if scores match
+ assertEquals("Doc " + id + " scores don't match\n"
+ CheckHits.topdocsString(top1,0,0)
+ CheckHits.topdocsString(top2,0,0)
- + "for query:" + q2.toString());
+ + "for query:" + q.toString(),
+ score, otherScore, CheckHits.explainToleranceDelta(score, otherScore));
+ }
}
+
+ // check if subset
+ if (!found) fail("Doc " + id + " not found\n"
+ + CheckHits.topdocsString(top1,0,0)
+ + CheckHits.topdocsString(top2,0,0)
+ + "for query:" + q.toString());
}
- // System.out.println("Total hits:"+tot);
}
-
+ public void testRewriteCoord1() throws Exception {
+ final Similarity oldSimilarity = s.getSimilarity();
+ try {
+ s.setSimilarity(new DefaultSimilarity() {
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return overlap / ((float)maxOverlap + 1);
+ }
+ });
+ BooleanQuery q1 = new BooleanQuery();
+ q1.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
+ BooleanQuery q2 = new BooleanQuery();
+ q2.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
+ q2.setMinimumNumberShouldMatch(1);
+ TopDocs top1 = s.search(q1,null,100);
+ TopDocs top2 = s.search(q2,null,100);
+ assertSubsetOfSameScores(q2, top1, top2);
+ } finally {
+ s.setSimilarity(oldSimilarity);
+ }
+ }
+
+ public void testRewriteNegate() throws Exception {
+ final Similarity oldSimilarity = s.getSimilarity();
+ try {
+ s.setSimilarity(new DefaultSimilarity() {
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return overlap / ((float)maxOverlap + 1);
+ }
+ });
+ BooleanQuery q1 = new BooleanQuery();
+ q1.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
+ BooleanQuery q2 = new BooleanQuery();
+ q2.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
+ q2.add(new TermQuery(new Term("data", "Z")), BooleanClause.Occur.MUST_NOT);
+ TopDocs top1 = s.search(q1,null,100);
+ TopDocs top2 = s.search(q2,null,100);
+ assertSubsetOfSameScores(q2, top1, top2);
+ } finally {
+ s.setSimilarity(oldSimilarity);
+ }
+ }
protected void printHits(String test, ScoreDoc[] h, IndexSearcher searcher) throws Exception {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java Mon Aug 13 11:16:57 2012
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.*;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Terms;
@@ -179,9 +178,8 @@ class MultiThreadTermVectorsReader imple
}
private void verifyVectors(Fields vectors, int num) throws IOException {
- FieldsEnum fieldsEnum = vectors.iterator();
- while(fieldsEnum.next() != null) {
- Terms terms = fieldsEnum.terms();
+ for (String field : vectors) {
+ Terms terms = vectors.terms(field);
assert terms != null;
verifyVector(terms.iterator(null), num);
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java Mon Aug 13 11:16:57 2012
@@ -59,6 +59,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
import org.junit.BeforeClass;
@@ -810,7 +811,7 @@ public class TestSort extends LuceneTest
assertMatches (full, queryG, sort, "ZYXW");
// Do the same for a ParallelMultiSearcher
- ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8));
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testEmptyFieldSort"));
IndexSearcher parallelSearcher=new IndexSearcher (full.getIndexReader(), exec);
sort.setSort (new SortField ("int", SortField.Type.INT),
@@ -852,7 +853,7 @@ public class TestSort extends LuceneTest
// test a variety of sorts using a parallel multisearcher
public void testParallelMultiSort() throws Exception {
- ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8));
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testParallelMultiSort"));
IndexSearcher searcher = new IndexSearcher(
new MultiReader(searchX.getIndexReader(),
searchY.getIndexReader()), exec);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java Mon Aug 13 11:16:57 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -124,13 +125,13 @@ public class TestTermVectors extends Luc
assertEquals(4, v.size());
String[] expectedFields = new String[]{"a", "b", "c", "x"};
int[] expectedPositions = new int[]{1, 2, 0};
- FieldsEnum fieldsEnum = v.iterator();
+ Iterator<String> fieldsEnum = v.iterator();
for(int i=0;i<expectedFields.length;i++) {
assertEquals(expectedFields[i], fieldsEnum.next());
assertEquals(3, v.terms(expectedFields[i]).size());
DocsAndPositionsEnum dpEnum = null;
- Terms terms = fieldsEnum.terms();
+ Terms terms = v.terms(expectedFields[i]);
assertNotNull(terms);
TermsEnum termsEnum = terms.iterator(null);
assertEquals("content", termsEnum.next().utf8ToString());
@@ -251,11 +252,11 @@ public class TestTermVectors extends Luc
writer.close();
IndexSearcher knownSearcher = newSearcher(reader);
knownSearcher.setSimilarity(new DefaultSimilarity());
- FieldsEnum fields = MultiFields.getFields(knownSearcher.reader).iterator();
+ Fields fields = MultiFields.getFields(knownSearcher.reader);
DocsEnum docs = null;
- while(fields.next() != null) {
- Terms terms = fields.terms();
+ for (String fieldName : fields) {
+ Terms terms = fields.terms(fieldName);
assertNotNull(terms); // NOTE: kinda sketchy assumptions, but ideally we would fix fieldsenum api...
TermsEnum termsEnum = terms.iterator(null);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Mon Aug 13 11:16:57 2012
@@ -160,7 +160,7 @@ public class MultiSpansWrapper extends S
}
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
if (current == null) {
return false;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java Mon Aug 13 11:16:57 2012
@@ -128,9 +128,10 @@ public class TestWeakIdentityMap extends
assertNotNull(it.next());
c++;
}
- assertTrue(size >= c);
- assertTrue(c >= map.size());
- size = map.size();
+ final int newSize = map.size();
+ assertTrue("previousSize("+size+")>=iteratorSize("+c+")", size >= c);
+ assertTrue("iteratorSize("+c+")>=newSize("+newSize+")", c >= newSize);
+ size = newSize;
} catch (InterruptedException ie) {}
map.clear();
@@ -159,7 +160,7 @@ public class TestWeakIdentityMap extends
public void testConcurrentHashMap() throws Exception {
// don't make threadCount and keyCount random, otherwise easily OOMs or fails otherwise:
final int threadCount = 8, keyCount = 1024;
- final ExecutorService exec = Executors.newFixedThreadPool(threadCount);
+ final ExecutorService exec = Executors.newFixedThreadPool(threadCount, new NamedThreadFactory("testConcurrentHashMap"));
final WeakIdentityMap<Object,Integer> map =
WeakIdentityMap.newConcurrentHashMap();
// we keep strong references to the keys,
@@ -228,9 +229,10 @@ public class TestWeakIdentityMap extends
assertNotNull(it.next());
c++;
}
- assertTrue(size >= c);
- assertTrue(c >= map.size());
- size = map.size();
+ final int newSize = map.size();
+ assertTrue("previousSize("+size+")>=iteratorSize("+c+")", size >= c);
+ assertTrue("iteratorSize("+c+")>=newSize("+newSize+")", c >= newSize);
+ size = newSize;
} catch (InterruptedException ie) {}
}