You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/08/13 16:18:32 UTC
svn commit: r1372438 [3/5] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/solr/
dev-tools/maven/solr/contrib/analysis-extras/
dev-tools/maven/solr/contrib/clusteri...
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon Aug 13 14:18:26 2012
@@ -685,12 +685,7 @@ public class CheckIndex {
DocsAndPositionsEnum postings = null;
String lastField = null;
- final FieldsEnum fieldsEnum = fields.iterator();
- while(true) {
- final String field = fieldsEnum.next();
- if (field == null) {
- break;
- }
+ for (String field : fields) {
// MultiFieldsEnum relies upon this order...
if (lastField != null && field.compareTo(lastField) <= 0) {
throw new RuntimeException("fields out of order: lastField=" + lastField + " field=" + field);
@@ -713,11 +708,16 @@ public class CheckIndex {
// assert fields.terms(field) != null;
computedFieldCount++;
- final Terms terms = fieldsEnum.terms();
+ final Terms terms = fields.terms(field);
if (terms == null) {
continue;
}
+ final boolean hasPositions = terms.hasPositions();
+ final boolean hasOffsets = terms.hasOffsets();
+ // term vectors cannot omit TF
+ final boolean hasFreqs = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+
final TermsEnum termsEnum = terms.iterator(null);
boolean hasOrd = true;
@@ -777,17 +777,10 @@ public class CheckIndex {
status.termCount++;
final DocsEnum docs2;
- final boolean hasPositions;
- // if we are checking vectors, we have freqs implicitly
- final boolean hasFreqs = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- // if we are checking vectors, offsets are a free-for-all anyway
- final boolean hasOffsets = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
if (postings != null) {
docs2 = postings;
- hasPositions = true;
} else {
docs2 = docs;
- hasPositions = false;
}
int lastDoc = -1;
@@ -824,22 +817,17 @@ public class CheckIndex {
if (hasPositions) {
for(int j=0;j<freq;j++) {
final int pos = postings.nextPosition();
- // NOTE: pos=-1 is allowed because of ancient bug
- // (LUCENE-1542) whereby IndexWriter could
- // write pos=-1 when first token's posInc is 0
- // (separately: analyzers should not give
- // posInc=0 to first token); also, term
- // vectors are allowed to return pos=-1 if
- // they indexed offset but not positions:
- if (pos < -1) {
+
+ if (pos < 0) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
if (pos < lastPos) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
lastPos = pos;
- if (postings.hasPayload()) {
- postings.getPayload();
+ BytesRef payload = postings.getPayload();
+ if (payload != null && payload.length < 1) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
}
if (hasOffsets) {
int startOffset = postings.startOffset();
@@ -924,14 +912,8 @@ public class CheckIndex {
int lastOffset = 0;
for(int posUpto=0;posUpto<freq;posUpto++) {
final int pos = postings.nextPosition();
- // NOTE: pos=-1 is allowed because of ancient bug
- // (LUCENE-1542) whereby IndexWriter could
- // write pos=-1 when first token's posInc is 0
- // (separately: analyzers should not give
- // posInc=0 to first token); also, term
- // vectors are allowed to return pos=-1 if
- // they indexed offset but not positions:
- if (pos < -1) {
+
+ if (pos < 0) {
throw new RuntimeException("position " + pos + " is out of bounds");
}
if (pos < lastPosition) {
@@ -1000,11 +982,7 @@ public class CheckIndex {
// only happen if it's a ghost field (field with
// no terms, eg there used to be terms but all
// docs got deleted and then merged away):
- // make sure TermsEnum is empty:
- final Terms fieldTerms2 = fieldsEnum.terms();
- if (fieldTerms2 != null && fieldTerms2.iterator(null).next() != null) {
- throw new RuntimeException("Fields.terms(field=" + field + ") returned null yet the field appears to have terms");
- }
+
} else {
if (fieldTerms instanceof BlockTreeTermsReader.FieldReader) {
final BlockTreeTermsReader.Stats stats = ((BlockTreeTermsReader.FieldReader) fieldTerms).computeStats();
@@ -1415,9 +1393,7 @@ public class CheckIndex {
status.docCount++;
}
- FieldsEnum fieldsEnum = tfv.iterator();
- String field = null;
- while((field = fieldsEnum.next()) != null) {
+ for(String field : tfv) {
if (doStats) {
status.totVectors++;
}
@@ -1432,6 +1408,8 @@ public class CheckIndex {
Terms terms = tfv.terms(field);
termsEnum = terms.iterator(termsEnum);
final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ final boolean postingsHasPayload = fieldInfo.hasPayloads();
+ final boolean vectorsHasPayload = terms.hasPayloads();
Terms postingsTerms = postingsFields.terms(field);
if (postingsTerms == null) {
@@ -1439,19 +1417,18 @@ public class CheckIndex {
}
postingsTermsEnum = postingsTerms.iterator(postingsTermsEnum);
+ final boolean hasProx = terms.hasOffsets() || terms.hasPositions();
BytesRef term = null;
while ((term = termsEnum.next()) != null) {
-
- final boolean hasProx;
- // Try positions:
- postings = termsEnum.docsAndPositions(null, postings);
- if (postings == null) {
- hasProx = false;
- // Try docIDs & freqs:
- docs = termsEnum.docs(null, docs);
+ if (hasProx) {
+ postings = termsEnum.docsAndPositions(null, postings);
+ assert postings != null;
+ docs = null;
} else {
- hasProx = true;
+ docs = termsEnum.docs(null, docs);
+ assert docs != null;
+ postings = null;
}
final DocsEnum docs2;
@@ -1504,7 +1481,7 @@ public class CheckIndex {
int pos = postings.nextPosition();
if (postingsPostings != null) {
int postingsPos = postingsPostings.nextPosition();
- if (pos != -1 && postingsPos != -1 && pos != postingsPos) {
+ if (terms.hasPositions() && pos != postingsPos) {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": pos=" + pos + " differs from postings pos=" + postingsPos);
}
}
@@ -1535,6 +1512,34 @@ public class CheckIndex {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
}
}
+
+ BytesRef payload = postings.getPayload();
+
+ if (payload != null) {
+ assert vectorsHasPayload;
+ }
+
+ if (postingsHasPayload && vectorsHasPayload) {
+ assert postingsPostings != null;
+
+ if (payload == null) {
+ // we have payloads, but not at this position.
+ // postings has payloads too, it should not have one at this position
+ if (postingsPostings.getPayload() != null) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsPostings.getPayload());
+ }
+ } else {
+ // we have payloads, and one at this position
+ // postings should also have one at this position, with the same bytes.
+ if (postingsPostings.getPayload() == null) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not.");
+ }
+ BytesRef postingsPayload = postingsPostings.getPayload();
+ if (!payload.equals(postingsPayload)) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but differs from postings payload=" + postingsPayload);
+ }
+ }
+ }
}
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java Mon Aug 13 14:18:26 2012
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.search.Query;
-import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.MergedIterator;
import org.apache.lucene.index.BufferedDeletesStream.QueryAndLimit;
class CoalescedDeletes {
@@ -48,13 +48,14 @@ class CoalescedDeletes {
public Iterable<Term> termsIterable() {
return new Iterable<Term>() {
+ @SuppressWarnings("unchecked")
@Override
public Iterator<Term> iterator() {
- ArrayList<Iterator<Term>> subs = new ArrayList<Iterator<Term>>(iterables.size());
- for (Iterable<Term> iterable : iterables) {
- subs.add(iterable.iterator());
+ Iterator<Term> subs[] = new Iterator[iterables.size()];
+ for (int i = 0; i < iterables.size(); i++) {
+ subs[i] = iterables.get(i).iterator();
}
- return mergedIterator(subs);
+ return new MergedIterator<Term>(subs);
}
};
}
@@ -86,106 +87,4 @@ class CoalescedDeletes {
}
};
}
-
- /** provides a merged view across multiple iterators */
- static Iterator<Term> mergedIterator(final List<Iterator<Term>> iterators) {
- return new Iterator<Term>() {
- Term current;
- TermMergeQueue queue = new TermMergeQueue(iterators.size());
- SubIterator[] top = new SubIterator[iterators.size()];
- int numTop;
-
- {
- int index = 0;
- for (Iterator<Term> iterator : iterators) {
- if (iterator.hasNext()) {
- SubIterator sub = new SubIterator();
- sub.current = iterator.next();
- sub.iterator = iterator;
- sub.index = index++;
- queue.add(sub);
- }
- }
- }
-
- public boolean hasNext() {
- if (queue.size() > 0) {
- return true;
- }
-
- for (int i = 0; i < numTop; i++) {
- if (top[i].iterator.hasNext()) {
- return true;
- }
- }
- return false;
- }
-
- public Term next() {
- // restore queue
- pushTop();
-
- // gather equal top fields
- if (queue.size() > 0) {
- pullTop();
- } else {
- current = null;
- }
- return current;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- private void pullTop() {
- // extract all subs from the queue that have the same top term
- assert numTop == 0;
- while (true) {
- top[numTop++] = queue.pop();
- if (queue.size() == 0
- || !(queue.top()).current.equals(top[0].current)) {
- break;
- }
- }
- current = top[0].current;
- }
-
- private void pushTop() {
- // call next() on each top, and put back into queue
- for (int i = 0; i < numTop; i++) {
- if (top[i].iterator.hasNext()) {
- top[i].current = top[i].iterator.next();
- queue.add(top[i]);
- } else {
- // no more terms
- top[i].current = null;
- }
- }
- numTop = 0;
- }
- };
- }
-
- private static class SubIterator {
- Iterator<Term> iterator;
- Term current;
- int index;
- }
-
- private static class TermMergeQueue extends PriorityQueue<SubIterator> {
- TermMergeQueue(int size) {
- super(size);
- }
-
- @Override
- protected boolean lessThan(SubIterator a, SubIterator b) {
- final int cmp = a.current.compareTo(b.current);
- if (cmp != 0) {
- return cmp < 0;
- } else {
- return a.index < b.index;
- }
- }
- }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocValues.java Mon Aug 13 14:18:26 2012
@@ -105,7 +105,7 @@ public abstract class DocValues implemen
* <p>
* {@link Source} instances obtained from this method are closed / released
* from the cache once this {@link DocValues} instance is closed by the
- * {@link IndexReader}, {@link Fields} or {@link FieldsEnum} the
+ * {@link IndexReader}, {@link Fields} or the
* {@link DocValues} was created from.
*/
public Source getSource() throws IOException {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Mon Aug 13 14:18:26 2012
@@ -48,11 +48,8 @@ public abstract class DocsAndPositionsEn
public abstract int endOffset() throws IOException;
/** Returns the payload at this position, or null if no
- * payload was indexed. Only call this once per
- * position. You should not modify anything (neither
- * members of the returned BytesRef nor bytes in the
- * byte[]). */
+ * payload was indexed. You should not modify anything
+ * (neither members of the returned BytesRef nor bytes
+ * in the byte[]). */
public abstract BytesRef getPayload() throws IOException;
-
- public abstract boolean hasPayload();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Fields.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Fields.java Mon Aug 13 14:18:26 2012
@@ -18,15 +18,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Iterator;
/** Flex API for access to fields and terms
* @lucene.experimental */
-public abstract class Fields {
+public abstract class Fields implements Iterable<String> {
/** Returns an iterator that will step through all fields
* names. This will not return null. */
- public abstract FieldsEnum iterator() throws IOException;
+ public abstract Iterator<String> iterator();
/** Get the {@link Terms} for this field. This will return
* null if the field does not exist. */
@@ -45,12 +46,7 @@ public abstract class Fields {
// TODO: deprecate?
public long getUniqueTermCount() throws IOException {
long numTerms = 0;
- FieldsEnum it = iterator();
- while(true) {
- String field = it.next();
- if (field == null) {
- break;
- }
+ for (String field : this) {
Terms terms = terms(field);
if (terms != null) {
final long termCount = terms.size();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Mon Aug 13 14:18:26 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.util.automaton.
import java.io.IOException;
import java.util.Comparator;
+import java.util.Iterator;
/** A <code>FilterAtomicReader</code> contains another AtomicReader, which it
* uses as its basic source of data, possibly transforming the data along the
@@ -46,7 +47,7 @@ public class FilterAtomicReader extends
}
@Override
- public FieldsEnum iterator() throws IOException {
+ public Iterator<String> iterator() {
return in.iterator();
}
@@ -109,28 +110,20 @@ public class FilterAtomicReader extends
public TermsEnum intersect(CompiledAutomaton automaton, BytesRef bytes) throws java.io.IOException {
return in.intersect(automaton, bytes);
}
- }
-
- /** Base class for filtering {@link TermsEnum} implementations. */
- public static class FilterFieldsEnum extends FieldsEnum {
- protected final FieldsEnum in;
- public FilterFieldsEnum(FieldsEnum in) {
- this.in = in;
- }
@Override
- public String next() throws IOException {
- return in.next();
+ public boolean hasOffsets() {
+ return in.hasOffsets();
}
@Override
- public Terms terms() throws IOException {
- return in.terms();
+ public boolean hasPositions() {
+ return in.hasPositions();
}
@Override
- public AttributeSource attributes() {
- return in.attributes();
+ public boolean hasPayloads() {
+ return in.hasPayloads();
}
}
@@ -292,11 +285,6 @@ public class FilterAtomicReader extends
public BytesRef getPayload() throws IOException {
return in.getPayload();
}
-
- @Override
- public boolean hasPayload() {
- return in.hasPayload();
- }
@Override
public AttributeSource attributes() {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Mon Aug 13 14:18:26 2012
@@ -42,6 +42,9 @@ public interface IndexableFieldType {
/** True if term vector positions should be indexed */
public boolean storeTermVectorPositions();
+
+ /** True if term vector payloads should be indexed */
+ public boolean storeTermVectorPayloads();
/** True if norms should not be indexed */
public boolean omitNorms();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeState.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeState.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeState.java Mon Aug 13 14:18:26 2012
@@ -199,6 +199,7 @@ public class MergeState {
// and we could make a codec(wrapper) to do all of this privately so IW is uninvolved
public PayloadProcessorProvider payloadProcessorProvider;
public ReaderPayloadProcessor[] readerPayloadProcessor;
+ public ReaderPayloadProcessor currentReaderPayloadProcessor;
public PayloadProcessor[] currentPayloadProcessor;
// TODO: get rid of this? it tells you which segments are 'aligned' (e.g. for bulk merging)
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java Mon Aug 13 14:18:26 2012
@@ -138,11 +138,6 @@ public final class MultiDocsAndPositions
}
@Override
- public boolean hasPayload() {
- return current.hasPayload();
- }
-
- @Override
public BytesRef getPayload() throws IOException {
return current.getPayload();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiFields.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiFields.java Mon Aug 13 14:18:26 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Collection;
@@ -27,6 +28,7 @@ import java.util.concurrent.ConcurrentHa
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.MergedIterator;
/**
* Exposes flex API, merged from flex API of sub-segments.
@@ -125,7 +127,7 @@ public final class MultiFields extends F
* term. This will return null if the field or term does
* not exist. */
public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
- return getTermDocsEnum(r, liveDocs, field, term);
+ return getTermDocsEnum(r, liveDocs, field, term, DocsEnum.FLAG_FREQS);
}
/** Returns {@link DocsEnum} for the specified field &
@@ -180,22 +182,14 @@ public final class MultiFields extends F
this.subSlices = subSlices;
}
+ @SuppressWarnings("unchecked")
@Override
- public FieldsEnum iterator() throws IOException {
-
- final List<FieldsEnum> fieldsEnums = new ArrayList<FieldsEnum>();
- final List<ReaderSlice> fieldsSlices = new ArrayList<ReaderSlice>();
+ public Iterator<String> iterator() {
+ Iterator<String> subIterators[] = new Iterator[subs.length];
for(int i=0;i<subs.length;i++) {
- fieldsEnums.add(subs[i].iterator());
- fieldsSlices.add(subSlices[i]);
- }
- if (fieldsEnums.size() == 0) {
- return FieldsEnum.EMPTY;
- } else {
- return new MultiFieldsEnum(this,
- fieldsEnums.toArray(FieldsEnum.EMPTY_ARRAY),
- fieldsSlices.toArray(ReaderSlice.EMPTY_ARRAY));
+ subIterators[i] = subs[i].iterator();
}
+ return new MergedIterator<String>(subIterators);
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java Mon Aug 13 14:18:26 2012
@@ -37,12 +37,19 @@ public final class MultiTerms extends Te
private final Terms[] subs;
private final ReaderSlice[] subSlices;
private final Comparator<BytesRef> termComp;
+ private final boolean hasOffsets;
+ private final boolean hasPositions;
+ private final boolean hasPayloads;
public MultiTerms(Terms[] subs, ReaderSlice[] subSlices) throws IOException {
this.subs = subs;
this.subSlices = subSlices;
Comparator<BytesRef> _termComp = null;
+ assert subs.length > 0 : "inefficient: don't use MultiTerms over one sub";
+ boolean _hasOffsets = true;
+ boolean _hasPositions = true;
+ boolean _hasPayloads = false;
for(int i=0;i<subs.length;i++) {
if (_termComp == null) {
_termComp = subs[i].getComparator();
@@ -54,9 +61,15 @@ public final class MultiTerms extends Te
throw new IllegalStateException("sub-readers have different BytesRef.Comparators; cannot merge");
}
}
+ _hasOffsets &= subs[i].hasOffsets();
+ _hasPositions &= subs[i].hasPositions();
+ _hasPayloads |= subs[i].hasPayloads();
}
termComp = _termComp;
+ hasOffsets = _hasOffsets;
+ hasPositions = _hasPositions;
+ hasPayloads = hasPositions && _hasPayloads; // if all subs have pos, and at least one has payloads.
}
@Override
@@ -142,5 +155,20 @@ public final class MultiTerms extends Te
public Comparator<BytesRef> getComparator() {
return termComp;
}
+
+ @Override
+ public boolean hasOffsets() {
+ return hasOffsets;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return hasPositions;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Mon Aug 13 14:18:26 2012
@@ -27,6 +27,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.UnmodifiableIterator;
/** An {@link AtomicReader} which reads multiple, parallel indexes. Each index
@@ -121,12 +122,10 @@ public final class ParallelAtomicReader
for (final AtomicReader reader : this.parallelReaders) {
final Fields readerFields = reader.fields();
if (readerFields != null) {
- final FieldsEnum it = readerFields.iterator();
- String name;
- while ((name = it.next()) != null) {
+ for (String field : readerFields) {
// only add if the reader responsible for that field name is the current:
- if (fieldToReader.get(name) == reader) {
- this.fields.addField(name, it.terms());
+ if (fieldToReader.get(field) == reader) {
+ this.fields.addField(field, readerFields.terms(field));
}
}
}
@@ -151,33 +150,6 @@ public final class ParallelAtomicReader
return buffer.append(')').toString();
}
- private final class ParallelFieldsEnum extends FieldsEnum {
- private String currentField;
- private final Iterator<String> keys;
- private final ParallelFields fields;
-
- ParallelFieldsEnum(ParallelFields fields) {
- this.fields = fields;
- keys = fields.fields.keySet().iterator();
- }
-
- @Override
- public String next() {
- if (keys.hasNext()) {
- currentField = keys.next();
- } else {
- currentField = null;
- }
- return currentField;
- }
-
- @Override
- public Terms terms() {
- return fields.terms(currentField);
- }
-
- }
-
// Single instance of this, per ParallelReader instance
private final class ParallelFields extends Fields {
final Map<String,Terms> fields = new TreeMap<String,Terms>();
@@ -190,8 +162,8 @@ public final class ParallelAtomicReader
}
@Override
- public FieldsEnum iterator() {
- return new ParallelFieldsEnum(this);
+ public Iterator<String> iterator() {
+ return new UnmodifiableIterator<String>(fields.keySet().iterator());
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Mon Aug 13 14:18:26 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -36,9 +37,12 @@ final class TermVectorsConsumerPerField
boolean doVectors;
boolean doVectorPositions;
boolean doVectorOffsets;
+ boolean doVectorPayloads;
int maxNumPostings;
OffsetAttribute offsetAttribute;
+ PayloadAttribute payloadAttribute;
+ boolean hasPayloads; // if enabled, and we actually saw any for this field
public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
this.termsHashPerField = termsHashPerField;
@@ -58,13 +62,46 @@ final class TermVectorsConsumerPerField
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
+ doVectorPayloads = false;
+ hasPayloads = false;
for(int i=0;i<count;i++) {
IndexableField field = fields[i];
- if (field.fieldType().storeTermVectors()) {
- doVectors = true;
- doVectorPositions |= field.fieldType().storeTermVectorPositions();
- doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
+ if (field.fieldType().indexed()) {
+ if (field.fieldType().storeTermVectors()) {
+ doVectors = true;
+ doVectorPositions |= field.fieldType().storeTermVectorPositions();
+ doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
+ if (doVectorPositions) {
+ doVectorPayloads |= field.fieldType().storeTermVectorPayloads();
+ } else if (field.fieldType().storeTermVectorPayloads()) {
+ // TODO: move this check somewhere else, and impl the other missing ones
+ throw new IllegalArgumentException("cannot index term vector payloads for field: " + field + " without term vector positions");
+ }
+ } else {
+ if (field.fieldType().storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("cannot index term vector offsets when term vectors are not indexed (field=\"" + field.name());
+ }
+ if (field.fieldType().storeTermVectorPositions()) {
+ throw new IllegalArgumentException("cannot index term vector positions when term vectors are not indexed (field=\"" + field.name());
+ }
+ if (field.fieldType().storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("cannot index term vector payloads when term vectors are not indexed (field=\"" + field.name());
+ }
+ }
+ } else {
+ if (field.fieldType().storeTermVectors()) {
+ throw new IllegalArgumentException("cannot index term vectors when field is not indexed (field=\"" + field.name());
+ }
+ if (field.fieldType().storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("cannot index term vector offsets when field is not indexed (field=\"" + field.name());
+ }
+ if (field.fieldType().storeTermVectorPositions()) {
+ throw new IllegalArgumentException("cannot index term vector positions when field is not indexed (field=\"" + field.name());
+ }
+ if (field.fieldType().storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("cannot index term vector payloads when field is not indexed (field=\"" + field.name());
+ }
}
}
@@ -121,7 +158,7 @@ final class TermVectorsConsumerPerField
final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
- tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
+ tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
final ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
final ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
@@ -165,52 +202,64 @@ final class TermVectorsConsumerPerField
} else {
offsetAttribute = null;
}
+ if (doVectorPayloads && fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
+ payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
+ } else {
+ payloadAttribute = null;
+ }
}
-
- @Override
- void newTerm(final int termID) {
- assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
-
- postings.freqs[termID] = 1;
-
+
+ void writeProx(TermVectorsPostingsArray postings, int termID) {
if (doVectorOffsets) {
int startOffset = fieldState.offset + offsetAttribute.startOffset();
int endOffset = fieldState.offset + offsetAttribute.endOffset();
- termsHashPerField.writeVInt(1, startOffset);
+ termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
termsHashPerField.writeVInt(1, endOffset - startOffset);
postings.lastOffsets[termID] = endOffset;
}
if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position);
+ final BytesRef payload;
+ if (payloadAttribute == null) {
+ payload = null;
+ } else {
+ payload = payloadAttribute.getPayload();
+ }
+
+ final int pos = fieldState.position - postings.lastPositions[termID];
+ if (payload != null && payload.length > 0) {
+ termsHashPerField.writeVInt(0, (pos<<1)|1);
+ termsHashPerField.writeVInt(0, payload.length);
+ termsHashPerField.writeBytes(0, payload.bytes, payload.offset, payload.length);
+ hasPayloads = true;
+ } else {
+ termsHashPerField.writeVInt(0, pos<<1);
+ }
postings.lastPositions[termID] = fieldState.position;
}
}
@Override
- void addTerm(final int termID) {
+ void newTerm(final int termID) {
+ assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
+ TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
- assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
+ postings.freqs[termID] = 1;
+ postings.lastOffsets[termID] = 0;
+ postings.lastPositions[termID] = 0;
+
+ writeProx(postings, termID);
+ }
+ @Override
+ void addTerm(final int termID) {
+ assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
postings.freqs[termID]++;
- if (doVectorOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
-
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
- postings.lastOffsets[termID] = endOffset;
- }
-
- if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
- postings.lastPositions[termID] = fieldState.position;
- }
+ writeProx(postings, termID);
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/Terms.java Mon Aug 13 14:18:26 2012
@@ -104,6 +104,15 @@ public abstract class Terms {
* measures, this measure does not take deleted documents
* into account. */
public abstract int getDocCount() throws IOException;
+
+ /** Returns true if documents in this field store offsets. */
+ public abstract boolean hasOffsets();
+
+ /** Returns true if documents in this field store positions. */
+ public abstract boolean hasPositions();
+
+ /** Returns true if documents in this field store payloads. */
+ public abstract boolean hasPayloads();
public final static Terms[] EMPTY_ARRAY = new Terms[0];
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Mon Aug 13 14:18:26 2012
@@ -213,7 +213,11 @@ public class BooleanQuery extends Query
}
public float coord(int overlap, int maxOverlap) {
- return similarity.coord(overlap, maxOverlap);
+ // LUCENE-4300: in most cases of maxOverlap=1, BQ rewrites itself away,
+ // so coord() is not applied. But when BQ cannot optimize itself away
+ // for a single clause (minNrShouldMatch, prohibited clauses, etc), its
+ // important not to apply coord(1,1) for consistency, it might not be 1.0F
+ return maxOverlap == 1 ? 1F : similarity.coord(overlap, maxOverlap);
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java Mon Aug 13 14:18:26 2012
@@ -41,16 +41,16 @@ class BooleanScorer2 extends Scorer {
private final List<Scorer> prohibitedScorers;
private class Coordinator {
- float[] coordFactors = null;
- int maxCoord = 0; // to be increased for each non prohibited scorer
- int nrMatchers; // to be increased by score() of match counting scorers.
-
- void init(boolean disableCoord) { // use after all scorers have been added.
+ final float coordFactors[];
+
+ Coordinator(int maxCoord, boolean disableCoord) {
coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1];
for (int i = 0; i < coordFactors.length; i++) {
coordFactors[i] = disableCoord ? 1.0f : ((BooleanWeight)weight).coord(i, maxCoord);
}
}
+
+ int nrMatchers; // to be increased by score() of match counting scorers.
}
private final Coordinator coordinator;
@@ -92,15 +92,13 @@ class BooleanScorer2 extends Scorer {
if (minNrShouldMatch < 0) {
throw new IllegalArgumentException("Minimum number of optional scorers should not be negative");
}
- coordinator = new Coordinator();
this.minNrShouldMatch = minNrShouldMatch;
- coordinator.maxCoord = maxCoord;
optionalScorers = optional;
requiredScorers = required;
prohibitedScorers = prohibited;
+ coordinator = new Coordinator(maxCoord, disableCoord);
- coordinator.init(disableCoord);
countingSumScorer = makeCountingSumScorer(disableCoord);
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java Mon Aug 13 14:18:26 2012
@@ -548,12 +548,7 @@ class UnionDocsAndPositionsEnum extends
@Override
public BytesRef getPayload() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasPayload() {
- throw new UnsupportedOperationException();
+ return null;
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Mon Aug 13 14:18:26 2012
@@ -118,8 +118,8 @@ public class PayloadTermQuery extends Sp
}
protected void processPayload(Similarity similarity) throws IOException {
- final DocsAndPositionsEnum postings = termSpans.getPostings();
- if (postings.hasPayload()) {
+ if (termSpans.isPayloadAvailable()) {
+ final DocsAndPositionsEnum postings = termSpans.getPostings();
payload = postings.getPayload();
if (payload != null) {
payloadScore = function.currentScore(doc, term.field(),
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Mon Aug 13 14:18:26 2012
@@ -126,7 +126,7 @@ public class NearSpansUnordered extends
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}
@@ -256,7 +256,7 @@ public class NearSpansUnordered extends
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
SpansCell pointer = min();
while (pointer != null) {
if (pointer.isPayloadAvailable()) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Mon Aug 13 14:18:26 2012
@@ -157,7 +157,7 @@ public class SpanNotQuery extends SpanQu
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return includeSpans.isPayloadAvailable();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Mon Aug 13 14:18:26 2012
@@ -247,7 +247,7 @@ public class SpanOrQuery extends SpanQue
}
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
Spans top = top();
return top != null && top.isPayloadAvailable();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Mon Aug 13 14:18:26 2012
@@ -176,7 +176,7 @@ public abstract class SpanPositionCheckQ
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
+ public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Mon Aug 13 14:18:26 2012
@@ -82,6 +82,6 @@ public abstract class Spans {
*
* @return true if there is a payload available at this position that can be loaded
*/
- public abstract boolean isPayloadAvailable();
+ public abstract boolean isPayloadAvailable() throws IOException;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Mon Aug 13 14:18:26 2012
@@ -36,6 +36,7 @@ public class TermSpans extends Spans {
protected int freq;
protected int count;
protected int position;
+ protected boolean readPayload;
public TermSpans(DocsAndPositionsEnum postings, Term term) {
this.postings = postings;
@@ -64,6 +65,7 @@ public class TermSpans extends Spans {
}
position = postings.nextPosition();
count++;
+ readPayload = false;
return true;
}
@@ -78,7 +80,7 @@ public class TermSpans extends Spans {
count = 0;
position = postings.nextPosition();
count++;
-
+ readPayload = false;
return true;
}
@@ -101,6 +103,7 @@ public class TermSpans extends Spans {
@Override
public Collection<byte[]> getPayload() throws IOException {
final BytesRef payload = postings.getPayload();
+ readPayload = true;
final byte[] bytes;
if (payload != null) {
bytes = new byte[payload.length];
@@ -113,8 +116,8 @@ public class TermSpans extends Spans {
// TODO: Remove warning after API has been finalized
@Override
- public boolean isPayloadAvailable() {
- return postings.hasPayload();
+ public boolean isPayloadAvailable() throws IOException {
+ return readPayload == false && postings.getPayload() != null;
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java Mon Aug 13 14:18:26 2012
@@ -62,7 +62,21 @@ public abstract class HashFunction imple
return loader.availableServices();
}
-
+ /**
+ * Reloads the hash function list from the given {@link ClassLoader}.
+ * Changes to the function list are visible after the method ends, all
+ * iterators ({@link #availableHashFunctionNames()},...) stay consistent.
+ *
+ * <p><b>NOTE:</b> Only new functions are added, existing ones are
+ * never removed or replaced.
+ *
+ * <p><em>This method is expensive and should only be called for discovery
+ * of new functions on the given classpath/classloader!</em>
+ */
+ public static void reloadHashFunctions(ClassLoader classloader) {
+ loader.reload(classloader);
+ }
+
@Override
public String toString() {
return name;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java Mon Aug 13 14:18:26 2012
@@ -28,14 +28,18 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
/**
* Test indexes ~82M docs with 26 terms each, so you get > Integer.MAX_VALUE terms/docs pairs
* @lucene.experimental
*/
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
+@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
public class Test2BPostings extends LuceneTestCase {
@Nightly
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Mon Aug 13 14:18:26 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Iterator;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -258,9 +259,10 @@ public class TestCodecs extends LuceneTe
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR));
- final FieldsEnum fieldsEnum = reader.iterator();
- assertNotNull(fieldsEnum.next());
- final Terms terms2 = fieldsEnum.terms();
+ final Iterator<String> fieldsEnum = reader.iterator();
+ String fieldName = fieldsEnum.next();
+ assertNotNull(fieldName);
+ final Terms terms2 = reader.terms(fieldName);
assertNotNull(terms2);
final TermsEnum termsEnum = terms2.iterator(null);
@@ -286,7 +288,7 @@ public class TestCodecs extends LuceneTe
assertEquals(termsEnum.seekCeil(new BytesRef(terms[i].text2)), TermsEnum.SeekStatus.FOUND);
}
- assertNull(fieldsEnum.next());
+ assertFalse(fieldsEnum.hasNext());
reader.close();
dir.close();
}
@@ -436,14 +438,14 @@ public class TestCodecs extends LuceneTe
final int pos = posEnum.nextPosition();
assertEquals(positions[i].pos, pos);
if (positions[i].payload != null) {
- assertTrue(posEnum.hasPayload());
+ assertNotNull(posEnum.getPayload());
if (random().nextInt(3) < 2) {
// Verify the payload bytes
final BytesRef otherPayload = posEnum.getPayload();
assertTrue("expected=" + positions[i].payload.toString() + " got=" + otherPayload.toString(), positions[i].payload.equals(otherPayload));
}
} else {
- assertFalse(posEnum.hasPayload());
+ assertNull(posEnum.getPayload());
}
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Mon Aug 13 14:18:26 2012
@@ -612,20 +612,20 @@ public void testFilesOpenClose() throws
}
// check dictionary and posting lists
- FieldsEnum fenum1 = MultiFields.getFields(index1).iterator();
- FieldsEnum fenum2 = MultiFields.getFields(index1).iterator();
- String field1 = null;
+ Fields fields1 = MultiFields.getFields(index1);
+ Fields fields2 = MultiFields.getFields(index2);
+ Iterator<String> fenum2 = fields2.iterator();
Bits liveDocs = MultiFields.getLiveDocs(index1);
- while((field1=fenum1.next()) != null) {
+ for (String field1 : fields1) {
assertEquals("Different fields", field1, fenum2.next());
- Terms terms1 = fenum1.terms();
+ Terms terms1 = fields1.terms(field1);
if (terms1 == null) {
- assertNull(fenum2.terms());
+ assertNull(fields2.terms(field1));
continue;
}
TermsEnum enum1 = terms1.iterator(null);
- Terms terms2 = fenum2.terms();
+ Terms terms2 = fields2.terms(field1);
assertNotNull(terms2);
TermsEnum enum2 = terms2.iterator(null);
@@ -644,6 +644,7 @@ public void testFilesOpenClose() throws
}
}
}
+ assertFalse(fenum2.hasNext());
}
public void testGetIndexCommit() throws IOException {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Mon Aug 13 14:18:26 2012
@@ -32,7 +32,6 @@ import java.util.LinkedList;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
@@ -236,10 +235,9 @@ public class TestDoc extends LuceneTestC
for (int i = 0; i < reader.numDocs(); i++)
out.println(reader.document(i));
- FieldsEnum fis = reader.fields().iterator();
- String field = fis.next();
- while(field != null) {
- Terms terms = fis.terms();
+ Fields fields = reader.fields();
+ for (String field : fields) {
+ Terms terms = fields.terms(field);
assertNotNull(terms);
TermsEnum tis = terms.iterator(null);
while(tis.next() != null) {
@@ -259,7 +257,6 @@ public class TestDoc extends LuceneTestC
out.println("");
}
}
- field = fis.next();
}
reader.close();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java Mon Aug 13 14:18:26 2012
@@ -61,9 +61,7 @@ public class TestDocCount extends Lucene
if (fields == null) {
return;
}
- FieldsEnum e = fields.iterator();
- String field;
- while ((field = e.next()) != null) {
+ for (String field : fields) {
Terms terms = fields.terms(field);
if (terms == null) {
continue;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon Aug 13 14:18:26 2012
@@ -205,11 +205,11 @@ public class TestDocumentWriter extends
int freq = termPositions.freq();
assertEquals(3, freq);
assertEquals(0, termPositions.nextPosition());
- assertEquals(true, termPositions.hasPayload());
+ assertNotNull(termPositions.getPayload());
assertEquals(6, termPositions.nextPosition());
- assertEquals(false, termPositions.hasPayload());
+ assertNull(termPositions.getPayload());
assertEquals(7, termPositions.nextPosition());
- assertEquals(false, termPositions.hasPayload());
+ assertNull(termPositions.getPayload());
reader.close();
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java Mon Aug 13 14:18:26 2012
@@ -27,12 +27,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeaks;
-
/**
* Tests for {@link DocumentsWriterStallControl}
*/
-@ThreadLeaks(failTestIfLeaking = true)
public class TestDocumentsWriterStallControl extends LuceneTestCase {
public void testSimpleStall() throws InterruptedException {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Mon Aug 13 14:18:26 2012
@@ -173,15 +173,15 @@ public class TestDuelingCodecs extends L
}
assertFieldStatistics(leftFields, rightFields);
- FieldsEnum leftEnum = leftFields.iterator();
- FieldsEnum rightEnum = rightFields.iterator();
+ Iterator<String> leftEnum = leftFields.iterator();
+ Iterator<String> rightEnum = rightFields.iterator();
- String field;
- while ((field = leftEnum.next()) != null) {
+ while (leftEnum.hasNext()) {
+ String field = leftEnum.next();
assertEquals(info, field, rightEnum.next());
- assertTerms(leftEnum.terms(), rightEnum.terms(), deep);
+ assertTerms(leftFields.terms(field), rightFields.terms(field), deep);
}
- assertNull(rightEnum.next());
+ assertFalse(rightEnum.hasNext());
}
/**
@@ -207,6 +207,9 @@ public class TestDuelingCodecs extends L
return;
}
assertTermsStatistics(leftTerms, rightTerms);
+ assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets());
+ assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions());
+ assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads());
TermsEnum leftTermsEnum = leftTerms.iterator(null);
TermsEnum rightTermsEnum = rightTerms.iterator(null);
@@ -409,12 +412,9 @@ public class TestDuelingCodecs extends L
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
+ assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
- if (leftDocs.hasPayload()) {
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- }
}
}
assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
@@ -506,10 +506,7 @@ public class TestDuelingCodecs extends L
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
- if (leftDocs.hasPayload()) {
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- }
+ assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
}
}
@@ -528,9 +525,7 @@ public class TestDuelingCodecs extends L
return;
}
- FieldsEnum fieldsEnum = leftFields.iterator();
- String field;
- while ((field = fieldsEnum.next()) != null) {
+ for (String field : leftFields) {
DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
if (leftNorms != null && rightNorms != null) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java Mon Aug 13 14:18:26 2012
@@ -40,10 +40,7 @@ public class TestFilterAtomicReader exte
TestFields(Fields in) {
super(in);
}
- @Override
- public FieldsEnum iterator() throws IOException {
- return new TestFieldsEnum(super.iterator());
- }
+
@Override
public Terms terms(String field) throws IOException {
return new TestTerms(super.terms(field));
@@ -61,17 +58,6 @@ public class TestFilterAtomicReader exte
}
}
- private static class TestFieldsEnum extends FilterFieldsEnum {
- TestFieldsEnum(FieldsEnum in) {
- super(in);
- }
-
- @Override
- public Terms terms() throws IOException {
- return new TestTerms(super.terms());
- }
- }
-
private static class TestTermsEnum extends FilterTermsEnum {
public TestTermsEnum(TermsEnum in) {
super(in);
@@ -194,7 +180,6 @@ public class TestFilterAtomicReader exte
checkOverrideMethods(FilterAtomicReader.class);
checkOverrideMethods(FilterAtomicReader.FilterFields.class);
checkOverrideMethods(FilterAtomicReader.FilterTerms.class);
- checkOverrideMethods(FilterAtomicReader.FilterFieldsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterTermsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterDocsEnum.class);
checkOverrideMethods(FilterAtomicReader.FilterDocsAndPositionsEnum.class);
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Aug 13 14:18:26 2012
@@ -526,8 +526,6 @@ public class TestIndexWriterOnDiskFull e
dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
final Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
try {
writer.addDocument(doc);
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Mon Aug 13 14:18:26 2012
@@ -60,17 +60,22 @@ public class TestIndexableField extends
@Override
public boolean storeTermVectors() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return indexed() && counter % 2 == 1 && counter % 10 != 9;
}
@Override
public boolean storeTermVectorOffsets() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return storeTermVectors() && counter % 10 != 9;
}
@Override
public boolean storeTermVectorPositions() {
- return counter % 2 == 1 && counter % 10 != 9;
+ return storeTermVectors() && counter % 10 != 9;
+ }
+
+ @Override
+ public boolean storeTermVectorPayloads() {
+ return storeTermVectors() && counter % 10 != 9;
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Mon Aug 13 14:18:26 2012
@@ -205,8 +205,11 @@ public class TestLongPostings extends Lu
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
- if (random().nextBoolean() && postings.hasPayload()) {
+ if (random().nextBoolean()) {
postings.getPayload();
+ if (random().nextBoolean()) {
+ postings.getPayload(); // get it again
+ }
}
}
}
@@ -247,8 +250,11 @@ public class TestLongPostings extends Lu
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
- if (random().nextBoolean() && postings.hasPayload()) {
+ if (random().nextBoolean()) {
postings.getPayload();
+ if (random().nextBoolean()) {
+ postings.getPayload(); // get it again
+ }
}
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java Mon Aug 13 14:18:26 2012
@@ -169,4 +169,22 @@ public class TestMultiFields extends Luc
r.close();
dir.close();
}
+
+ public void testTermDocsEnum() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document d = new Document();
+ d.add(newStringField("f", "j", Field.Store.NO));
+ w.addDocument(d);
+ w.commit();
+ w.addDocument(d);
+ IndexReader r = w.getReader();
+ w.close();
+ DocsEnum de = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j"));
+ assertEquals(0, de.nextDoc());
+ assertEquals(1, de.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java Mon Aug 13 14:18:26 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Iterator;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -94,20 +95,21 @@ public class TestParallelTermEnum extend
Bits liveDocs = pr.getLiveDocs();
- FieldsEnum fe = pr.fields().iterator();
+ Fields fields = pr.fields();
+ Iterator<String> fe = fields.iterator();
String f = fe.next();
assertEquals("field1", f);
- checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the");
+ checkTerms(fields.terms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
f = fe.next();
assertEquals("field2", f);
- checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the");
+ checkTerms(fields.terms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
f = fe.next();
assertEquals("field3", f);
- checkTerms(fe.terms(), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the");
+ checkTerms(fields.terms(f), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the");
- assertNull(fe.next());
+ assertFalse(fe.hasNext());
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Mon Aug 13 14:18:26 2012
@@ -145,8 +145,18 @@ public class TestPayloadProcessorProvide
Document doc = new Document();
doc.add(newField("id", "doc" + i, customType));
doc.add(newTextField("content", "doc content " + i, Field.Store.NO));
- doc.add(new TextField("p", payloadTS1));
- doc.add(new TextField("p", payloadTS2));
+ if (random.nextBoolean()) {
+ doc.add(new TextField("p", payloadTS1));
+ doc.add(new TextField("p", payloadTS2));
+ } else {
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setStoreTermVectors(true);
+ type.setStoreTermVectorPositions(true);
+ type.setStoreTermVectorPayloads(true);
+ type.setStoreTermVectorOffsets(random.nextBoolean());
+ doc.add(new Field("p", payloadTS1, type));
+ doc.add(new Field("p", payloadTS2, type));
+ }
writer.addDocument(doc);
if (multipleCommits && (i % 4 == 0)) {
writer.commit();
@@ -163,8 +173,8 @@ public class TestPayloadProcessorProvide
DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
tpe.nextPosition();
- if (tpe.hasPayload()) {
- BytesRef payload = tpe.getPayload();
+ BytesRef payload = tpe.getPayload();
+ if (payload != null) {
assertEquals(1, payload.length);
assertEquals(1, payload.bytes[0]);
++numPayloads;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java Mon Aug 13 14:18:26 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
+import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
@@ -200,18 +201,10 @@ public class TestPayloads extends Lucene
for (int i = 0; i < freq; i++) {
for (int j = 0; j < numTerms; j++) {
tps[j].nextPosition();
- if (tps[j].hasPayload()) {
- BytesRef br = tps[j].getPayload();
+ BytesRef br = tps[j].getPayload();
+ if (br != null) {
System.arraycopy(br.bytes, br.offset, verifyPayloadData, offset, br.length);
offset += br.length;
- // Just to ensure all codecs can
- // handle a caller that mucks with the
- // returned payload:
- if (rarely()) {
- br.bytes = new byte[random().nextInt(5)];
- }
- br.length = 0;
- br.offset = 0;
}
}
}
@@ -267,11 +260,6 @@ public class TestPayloads extends Lucene
tp.nextPosition();
assertEquals("Wrong payload length.", 3 * skipInterval - 2 * numDocs - 1, tp.getPayload().length);
- /*
- * Test multiple call of getPayload()
- */
- assertFalse(tp.hasPayload());
-
reader.close();
// test long payload
@@ -591,4 +579,73 @@ public class TestPayloads extends Lucene
dir.close();
}
+
+ /** some docs have payload att, some not */
+ public void testMixupDocs() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ Field field = new TextField("field", "", Field.Store.NO);
+ TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ doc.add(field);
+ writer.addDocument(doc);
+ Token withPayload = new Token("withPayload", 0, 11);
+ withPayload.setPayload(new BytesRef("test"));
+ ts = new CannedTokenStream(withPayload);
+ assertTrue(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ writer.addDocument(doc);
+ ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ writer.addDocument(doc);
+ DirectoryReader reader = writer.getReader();
+ AtomicReader sr = reader.getSequentialSubReaders().get(0);
+ DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
+ de.nextDoc();
+ de.nextPosition();
+ assertEquals(new BytesRef("test"), de.getPayload());
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
+ /** some field instances have payload att, some not */
+ public void testMixupMultiValued() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ Field field = new TextField("field", "", Field.Store.NO);
+ TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field.setTokenStream(ts);
+ doc.add(field);
+ Field field2 = new TextField("field", "", Field.Store.NO);
+ Token withPayload = new Token("withPayload", 0, 11);
+ withPayload.setPayload(new BytesRef("test"));
+ ts = new CannedTokenStream(withPayload);
+ assertTrue(ts.hasAttribute(PayloadAttribute.class));
+ field2.setTokenStream(ts);
+ doc.add(field2);
+ Field field3 = new TextField("field", "", Field.Store.NO);
+ ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+ assertFalse(ts.hasAttribute(PayloadAttribute.class));
+ field3.setTokenStream(ts);
+ doc.add(field3);
+ writer.addDocument(doc);
+ DirectoryReader reader = writer.getReader();
+ SegmentReader sr = getOnlySegmentReader(reader);
+ DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
+ de.nextDoc();
+ de.nextPosition();
+ assertEquals(new BytesRef("test"), de.getPayload());
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1372438&r1=1372437&r2=1372438&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Mon Aug 13 14:18:26 2012
@@ -23,8 +23,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeMap;
@@ -731,20 +733,21 @@ public class TestPostingsFormat extends
System.out.println(" now check payload length=" + (position.payload == null ? 0 : position.payload.length));
}
if (position.payload == null || position.payload.length == 0) {
- assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
+ assertNull("should not have payload", docsAndPositionsEnum.getPayload());
} else {
- assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
-
BytesRef payload = docsAndPositionsEnum.getPayload();
- assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
+ assertNotNull("should have payload but doesn't", payload);
- assertNotNull("payload should not be null", payload);
assertEquals("payload length is wrong", position.payload.length, payload.length);
for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
assertEquals("payload bytes are wrong",
position.payload[byteUpto],
payload.bytes[payload.offset+byteUpto]);
}
+
+ // make a deep copy
+ payload = BytesRef.deepCopyOf(payload);
+ assertEquals("2nd call to getPayload returns something different!", payload, docsAndPositionsEnum.getPayload());
}
} else {
if (VERBOSE) {
@@ -888,6 +891,26 @@ public class TestPostingsFormat extends
}
}
}
+
+ private void testFields(Fields fields) throws Exception {
+ Iterator<String> iterator = fields.iterator();
+ while (iterator.hasNext()) {
+ String field = iterator.next();
+ try {
+ iterator.remove();
+ fail("Fields.iterator() allows for removal");
+ } catch (UnsupportedOperationException expected) {
+ // expected;
+ }
+ }
+ assertFalse(iterator.hasNext());
+ try {
+ iterator.next();
+ fail("Fields.iterator() doesn't throw NoSuchElementException when past the end");
+ } catch (NoSuchElementException expected) {
+ // expected
+ }
+ }
public void test() throws Exception {
Directory dir = newFSDirectory(_TestUtil.getTempDir("testPostingsFormat"));
@@ -897,6 +920,7 @@ public class TestPostingsFormat extends
FieldsProducer fieldsProducer = buildIndex(dir, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads);
+ testFields(fieldsProducer);
//testTerms(fieldsProducer, EnumSet.noneOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.LIVE_DOCS), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.LIVE_DOCS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);