You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/22 07:44:21 UTC
svn commit: r1633538 [2/5] - in /lucene/dev/branches/lucene5969: ./ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/analysis/icu/
lucene/analysis/kuromoji/ lucene/analysis/morfologik/
lucene/analysis/phonetic/ lucene/analysis/smartcn/ lucene/an...
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Wed Oct 22 05:44:17 2014
@@ -284,12 +284,12 @@ public class FieldInfos implements Itera
// rather, each component in the chain should update
// what it "owns". EG fieldType.indexOptions() should
// be updated by maybe FreqProxTermsWriterPerField:
- return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
+ return addOrUpdateInternal(name, -1, false,
fieldType.omitNorms(), false,
fieldType.indexOptions(), fieldType.docValueType());
}
- private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
+ private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
FieldInfo fi = fieldInfo(name);
@@ -300,12 +300,12 @@ public class FieldInfos implements Itera
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
- fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
+ fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
assert !byName.containsKey(fi.name);
assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
} else {
- fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
+ fi.update(storeTermVector, omitNorms, storePayloads, indexOptions);
if (docValues != null) {
// only pay the synchronization cost if fi does not already have a DVType
@@ -323,7 +323,7 @@ public class FieldInfos implements Itera
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
- return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
+ return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType());
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java Wed Oct 22 05:44:17 2014
@@ -61,33 +61,10 @@ public abstract class FilterDirectoryRea
}
- /**
- * A no-op SubReaderWrapper that simply returns the parent
- * DirectoryReader's original subreaders.
- */
- public static class StandardReaderWrapper extends SubReaderWrapper {
-
- /** Constructor */
- public StandardReaderWrapper() {}
-
- @Override
- public LeafReader wrap(LeafReader reader) {
- return reader;
- }
- }
-
/** The filtered DirectoryReader */
protected final DirectoryReader in;
/**
- * Create a new FilterDirectoryReader that filters a passed in DirectoryReader.
- * @param in the DirectoryReader to filter
- */
- public FilterDirectoryReader(DirectoryReader in) {
- this(in, new StandardReaderWrapper());
- }
-
- /**
* Create a new FilterDirectoryReader that filters a passed in DirectoryReader,
* using the supplied SubReaderWrapper to wrap its subreader.
* @param in the DirectoryReader to filter
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Wed Oct 22 05:44:17 2014
@@ -18,18 +18,14 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.similarities.DefaultSimilarity; // javadocs
import org.apache.lucene.search.similarities.Similarity; // javadocs
-import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
-// TODO: we need to break out separate StoredField...
-
/** Represents a single field for indexing. IndexWriter
* consumes Iterable<IndexableField> as a document.
*
@@ -68,7 +64,7 @@ public interface IndexableField extends
* the range of that encoding.
* <p>
* It is illegal to return a boost other than 1.0f for a field that is not
- * indexed ({@link IndexableFieldType#indexed()} is false) or omits normalization values
+ * indexed ({@link IndexableFieldType#indexOptions()} is null) or omits normalization values
* ({@link IndexableFieldType#omitNorms()} returns true).
*
* @see Similarity#computeNorm(FieldInvertState)
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Wed Oct 22 05:44:17 2014
@@ -27,9 +27,6 @@ import org.apache.lucene.index.FieldInfo
*/
public interface IndexableFieldType {
- /** True if this field should be indexed (inverted) */
- public boolean indexed();
-
/** True if the field's value should be stored */
public boolean stored();
@@ -37,7 +34,7 @@ public interface IndexableFieldType {
* True if this field's value should be analyzed by the
* {@link Analyzer}.
* <p>
- * This has no effect if {@link #indexed()} returns false.
+ * This has no effect if {@link #indexOptions()} returns null.
*/
// TODO: shouldn't we remove this? Whether/how a field is
// tokenized is an impl detail under Field?
@@ -51,7 +48,7 @@ public interface IndexableFieldType {
* can be accessed in a document-oriented way from
* {@link IndexReader#getTermVector(int,String)}.
* <p>
- * This option is illegal if {@link #indexed()} returns false.
+ * This option is illegal if {@link #indexOptions()} returns null.
*/
public boolean storeTermVectors();
@@ -91,7 +88,8 @@ public interface IndexableFieldType {
public boolean omitNorms();
/** {@link IndexOptions}, describing what should be
- * recorded into the inverted index */
+ * recorded into the inverted index, or null if this field
+ * is not indexed */
public IndexOptions indexOptions();
/**
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Wed Oct 22 05:44:17 2014
@@ -113,7 +113,7 @@ final class TermVectorsConsumerPerField
@Override
boolean start(IndexableField field, boolean first) {
- assert field.fieldType().indexed();
+ assert field.fieldType().indexOptions() != null;
if (first) {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java Wed Oct 22 05:44:17 2014
@@ -20,7 +20,6 @@ import java.io.IOException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.OpenBitSet;
/**
* Base class for DocIdSet to be used with DocValues. The implementation
@@ -114,8 +113,8 @@ public abstract class DocValuesDocIdSet
return maxDoc;
}
};
- } else if (acceptDocs instanceof FixedBitSet || acceptDocs instanceof OpenBitSet) {
- // special case for FixedBitSet / OpenBitSet: use the iterator and filter it
+ } else if (acceptDocs instanceof FixedBitSet) {
+ // special case for FixedBitSet: use the iterator and filter it
// (used e.g. when Filters are chained by FilteredQuery)
return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) {
@Override
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java Wed Oct 22 05:44:17 2014
@@ -40,7 +40,7 @@ public final class DocIdSetBuilder {
/** Sole constructor. */
public DocIdSetBuilder(int maxDoc) {
this.maxDoc = maxDoc;
- threshold = maxDoc >>> 14;
+ threshold = maxDoc >>> 10;
}
/**
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Wed Oct 22 05:44:17 2014
@@ -59,24 +59,7 @@ public final class FixedBitSet extends D
@Override
public int nextDoc() {
- if (doc == NO_MORE_DOCS || ++doc >= numBits) {
- return doc = NO_MORE_DOCS;
- }
- int i = doc >> 6;
- long word = bits[i] >> doc; // skip all the bits to the right of index
-
- if (word != 0) {
- return doc = doc + Long.numberOfTrailingZeros(word);
- }
-
- while (++i < numWords) {
- word = bits[i];
- if (word != 0) {
- return doc = (i << 6) + Long.numberOfTrailingZeros(word);
- }
- }
-
- return doc = NO_MORE_DOCS;
+ return advance(doc + 1);
}
@Override
@@ -91,7 +74,7 @@ public final class FixedBitSet extends D
@Override
public int advance(int target) {
- if (doc == NO_MORE_DOCS || target >= numBits) {
+ if (target >= numBits) {
return doc = NO_MORE_DOCS;
}
int i = target >> 6;
@@ -325,13 +308,7 @@ public final class FixedBitSet extends D
/** Does in-place OR of the bits provided by the
* iterator. */
public void or(DocIdSetIterator iter) throws IOException {
- if (iter instanceof OpenBitSetIterator && iter.docID() == -1) {
- final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
- or(obs.arr, obs.words);
- // advance after last doc that would be accepted if standard
- // iteration is used (to exhaust it):
- obs.advance(numBits);
- } else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
+ if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
or(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard
@@ -381,13 +358,7 @@ public final class FixedBitSet extends D
/** Does in-place AND of the bits provided by the
* iterator. */
public void and(DocIdSetIterator iter) throws IOException {
- if (iter instanceof OpenBitSetIterator && iter.docID() == -1) {
- final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
- and(obs.arr, obs.words);
- // advance after last doc that would be accepted if standard
- // iteration is used (to exhaust it):
- obs.advance(numBits);
- } else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
+ if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
and(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard
@@ -435,13 +406,7 @@ public final class FixedBitSet extends D
/** Does in-place AND NOT of the bits provided by the
* iterator. */
public void andNot(DocIdSetIterator iter) throws IOException {
- if (iter instanceof OpenBitSetIterator && iter.docID() == -1) {
- final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
- andNot(obs.arr, obs.words);
- // advance after last doc that would be accepted if standard
- // iteration is used (to exhaust it):
- obs.advance(numBits);
- } else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
+ if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
andNot(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NotDocIdSet.java Wed Oct 22 05:44:17 2014
@@ -82,9 +82,6 @@ public final class NotDocIdSet extends D
@Override
public int nextDoc() throws IOException {
- if (doc == NO_MORE_DOCS) {
- return NO_MORE_DOCS;
- }
return advance(doc + 1);
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java Wed Oct 22 05:44:17 2014
@@ -143,8 +143,10 @@ public final class NumericUtils {
* @param bytes will contain the encoded value
*/
public static void longToPrefixCodedBytes(final long val, final int shift, final BytesRefBuilder bytes) {
- if ((shift & ~0x3f) != 0) // ensure shift is 0..63
- throw new IllegalArgumentException("Illegal shift value, must be 0..63");
+ // ensure shift is 0..63
+ if ((shift & ~0x3f) != 0) {
+ throw new IllegalArgumentException("Illegal shift value, must be 0..63; got shift=" + shift);
+ }
int nChars = (((63-shift)*37)>>8) + 1; // i/7 is the same as (i*37)>>8 for i in 0..63
bytes.setLength(nChars+1); // one extra for the byte that contains the shift info
bytes.grow(BUF_SIZE_LONG);
@@ -169,8 +171,10 @@ public final class NumericUtils {
* @param bytes will contain the encoded value
*/
public static void intToPrefixCodedBytes(final int val, final int shift, final BytesRefBuilder bytes) {
- if ((shift & ~0x1f) != 0) // ensure shift is 0..31
- throw new IllegalArgumentException("Illegal shift value, must be 0..31");
+ // ensure shift is 0..31
+ if ((shift & ~0x1f) != 0) {
+ throw new IllegalArgumentException("Illegal shift value, must be 0..31; got shift=" + shift);
+ }
int nChars = (((31-shift)*37)>>8) + 1; // i/7 is the same as (i*37)>>8 for i in 0..63
bytes.setLength(nChars+1); // one extra for the byte that contains the shift info
bytes.grow(NumericUtils.BUF_SIZE_LONG); // use the max
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java Wed Oct 22 05:44:17 2014
@@ -282,9 +282,6 @@ public class RoaringDocIdSet extends Doc
@Override
public int nextDoc() throws IOException {
- if (doc == NO_MORE_DOCS) {
- return NO_MORE_DOCS;
- }
final int subNext = sub.nextDoc();
if (subNext == NO_MORE_DOCS) {
return firstDocFromNextBlock();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java Wed Oct 22 05:44:17 2014
@@ -248,16 +248,17 @@ public class SparseFixedBitSet extends D
@Override
public int nextDoc() throws IOException {
- if (++doc >= length) {
- return doc = NO_MORE_DOCS;
- }
- return currentOrNextDoc();
+ return advance(doc + 1);
}
- private int currentOrNextDoc() {
- final int i4096 = doc >>> 12;
+ @Override
+ public int advance(int target) throws IOException {
+ final int i4096 = target >>> 12;
+ if (i4096 >= indices.length) {
+ return doc = NO_MORE_DOCS;
+ }
final long index = indices[i4096];
- int i64 = doc >>> 6;
+ int i64 = target >>> 6;
long indexBits = index >>> i64;
if (indexBits == 0) {
// if the index is zero, it means that there is no value in the
@@ -270,7 +271,7 @@ public class SparseFixedBitSet extends D
} else {
// We know we still have some 64-bits blocks that have bits set, let's
// advance to the next one by skipping trailing zeros of the index
- int i1 = doc & 0x3F;
+ int i1 = target & 0x3F;
int trailingZeros = Long.numberOfTrailingZeros(indexBits);
if (trailingZeros != 0) {
// no bits in the current long, go to the next one
@@ -285,7 +286,7 @@ public class SparseFixedBitSet extends D
int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
final long[] longArray = bits[i4096];
assert longArray[longIndex] != 0;
- long bits = SparseFixedBitSet.this.bits[i4096][longIndex] >>> i1; // shifts are mod 64 in java
+ long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
if (bits != 0L) {
// hurray, we found some non-zero bits, this gives us the next document:
i1 += Long.numberOfTrailingZeros(bits);
@@ -313,17 +314,6 @@ public class SparseFixedBitSet extends D
}
@Override
- public int advance(int target) throws IOException {
- if (target >= length) {
- return doc = NO_MORE_DOCS;
- } else {
- doc = target;
- }
-
- return currentOrNextDoc();
- }
-
- @Override
public long cost() {
// although constant-time, approximateCardinality is a bit expensive so
// we cache it to avoid performance traps eg. when sorting iterators by
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Wed Oct 22 05:44:17 2014
@@ -1376,16 +1376,6 @@ public final class FST<T> implements Acc
public abstract boolean reversed();
}
- private static class ArcAndState<T> {
- final Arc<T> arc;
- final IntsRef chain;
-
- public ArcAndState(Arc<T> arc, IntsRef chain) {
- this.arc = arc;
- this.chain = chain;
- }
- }
-
/*
public void countSingleChains() throws IOException {
// TODO: must assert this FST was built with
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Wed Oct 22 05:44:17 2014
@@ -815,22 +815,6 @@ public class PackedInts {
throw new AssertionError("Unknown Writer format: " + format);
}
}
-
- /**
- * Expert: Restore a {@link Reader} from a stream without reading metadata at
- * the beginning of the stream. This method is useful to restore data when
- * metadata has been previously read using {@link #readHeader(DataInput)}.
- *
- * @param in the stream to read data from, positioned at the beginning of the packed values
- * @param header metadata result from <code>readHeader()</code>
- * @return a Reader
- * @throws IOException If there is a low-level I/O error
- * @see #readHeader(DataInput)
- * @lucene.internal
- */
- public static Reader getReaderNoHeader(DataInput in, Header header) throws IOException {
- return getReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
- }
/**
* Restore a {@link Reader} from a stream.
@@ -943,23 +927,6 @@ public class PackedInts {
throw new AssertionError("Unknwown format: " + format);
}
}
-
- /**
- * Expert: Construct a direct {@link Reader} from an {@link IndexInput}
- * without reading metadata at the beginning of the stream. This method is
- * useful to restore data when metadata has been previously read using
- * {@link #readHeader(DataInput)}.
- *
- * @param in the stream to read data from, positioned at the beginning of the packed values
- * @param header metadata result from <code>readHeader()</code>
- * @return a Reader
- * @throws IOException If there is a low-level I/O error
- * @see #readHeader(DataInput)
- * @lucene.internal
- */
- public static Reader getDirectReaderNoHeader(IndexInput in, Header header) throws IOException {
- return getDirectReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
- }
/**
* Construct a direct {@link Reader} from an {@link IndexInput}. This method
@@ -1216,42 +1183,6 @@ public class PackedInts {
}
}
- /**
- * Expert: reads only the metadata from a stream. This is useful to later
- * restore a stream or open a direct reader via
- * {@link #getReaderNoHeader(DataInput, Header)}
- * or {@link #getDirectReaderNoHeader(IndexInput, Header)}.
- * @param in the stream to read data
- * @return packed integer metadata.
- * @throws IOException If there is a low-level I/O error
- * @see #getReaderNoHeader(DataInput, Header)
- * @see #getDirectReaderNoHeader(IndexInput, Header)
- */
- public static Header readHeader(DataInput in) throws IOException {
- final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
- final int bitsPerValue = in.readVInt();
- assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
- final int valueCount = in.readVInt();
- final Format format = Format.byId(in.readVInt());
- return new Header(format, valueCount, bitsPerValue, version);
- }
-
- /** Header identifying the structure of a packed integer array. */
- public static class Header {
-
- private final Format format;
- private final int valueCount;
- private final int bitsPerValue;
- private final int version;
-
- public Header(Format format, int valueCount, int bitsPerValue, int version) {
- this.format = format;
- this.valueCount = valueCount;
- this.bitsPerValue = bitsPerValue;
- this.version = version;
- }
- }
-
/** Check that the block size is a power of 2, in the right bounds, and return
* its log in base 2. */
static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/overview.html?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/overview.html (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/overview.html Wed Oct 22 05:44:17 2014
@@ -118,7 +118,7 @@ which implements files as memory-residen
<li>
<b>{@link org.apache.lucene.util}</b>
-contains a few handy data structures and util classes, ie {@link org.apache.lucene.util.OpenBitSet OpenBitSet}
+contains a few handy data structures and util classes, ie {@link org.apache.lucene.util.FixedBitSet FixedBitSet}
and {@link org.apache.lucene.util.PriorityQueue PriorityQueue}.</li>
</ul>
To use Lucene, an application should:
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Wed Oct 22 05:44:17 2014
@@ -305,7 +305,6 @@ public class TestMockAnalyzer extends Ba
final RandomIndexWriter writer = new RandomIndexWriter(random(), newDirectory());
final Document doc = new Document();
final FieldType ft = new FieldType();
- ft.setIndexed(true);
ft.setIndexOptions(IndexOptions.DOCS_ONLY);
ft.setTokenized(true);
ft.setStoreTermVectors(true);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Wed Oct 22 05:44:17 2014
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.StorableField;
@@ -64,7 +65,7 @@ public class TestDocument extends Lucene
assertTrue(binaryFld.binaryValue() != null);
assertTrue(binaryFld.fieldType().stored());
- assertFalse(binaryFld.fieldType().indexed());
+ assertNull(binaryFld.fieldType().indexOptions());
String binaryTest = doc.getBinaryValue("binary").utf8ToString();
assertTrue(binaryTest.equals(binaryVal));
@@ -263,7 +264,7 @@ public class TestDocument extends Lucene
FieldType stored = new FieldType();
stored.setStored(true);
FieldType indexedNotTokenized = new FieldType();
- indexedNotTokenized.setIndexed(true);
+ indexedNotTokenized.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
indexedNotTokenized.setTokenized(false);
doc.add(new StringField("keyword", "test1", Field.Store.YES));
doc.add(new StringField("keyword", "test2", Field.Store.YES));
@@ -271,10 +272,8 @@ public class TestDocument extends Lucene
doc.add(new TextField("text", "test2", Field.Store.YES));
doc.add(new Field("unindexed", "test1", stored));
doc.add(new Field("unindexed", "test2", stored));
- doc
- .add(new TextField("unstored", "test1", Field.Store.NO));
- doc
- .add(new TextField("unstored", "test2", Field.Store.NO));
+ doc.add(new TextField("unstored", "test1", Field.Store.NO));
+ doc.add(new TextField("unstored", "test2", Field.Store.NO));
doc.add(new Field("indexed_not_tokenized", "test1", indexedNotTokenized));
doc.add(new Field("indexed_not_tokenized", "test2", indexedNotTokenized));
return doc;
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java Wed Oct 22 05:44:17 2014
@@ -43,7 +43,7 @@ public class TestFieldType extends Lucen
assertFalse(ft4.equals(ft));
FieldType ft5 = new FieldType();
- ft5.setIndexed(true);
+ ft5.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
assertFalse(ft5.equals(ft));
FieldType ft6 = new FieldType();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java Wed Oct 22 05:44:17 2014
@@ -59,7 +59,6 @@ public class Test4GBStoredFields extends
final Document doc = new Document();
final FieldType ft = new FieldType();
- ft.setIndexed(false);
ft.setStored(true);
ft.freeze();
final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Wed Oct 22 05:44:17 2014
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -35,6 +34,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.PhraseQuery;
@@ -1142,7 +1142,7 @@ public class TestAddIndexes extends Luce
IndexWriter w = new IndexWriter(toAdd, conf);
Document doc = new Document();
FieldType customType = new FieldType();
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
doc.add(newField("foo", "bar", customType));
w.addDocument(doc);
w.close();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Wed Oct 22 05:44:17 2014
@@ -93,9 +93,6 @@ public class TestCodecs extends LuceneTe
fieldInfo = fieldInfos.addOrUpdate(name, new IndexableFieldType() {
@Override
- public boolean indexed() { return true; }
-
- @Override
public boolean stored() { return false; }
@Override
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Wed Oct 22 05:44:17 2014
@@ -268,7 +268,7 @@ public class TestConsistentFieldNumbers
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
- assertEquals(expected.fieldType().indexed(), fi.isIndexed());
+ assertEquals(expected.fieldType().indexOptions(), fi.getIndexOptions());
assertEquals(expected.fieldType().storeTermVectors(), fi.hasVectors());
}
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java Wed Oct 22 05:44:17 2014
@@ -17,17 +17,18 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
/**
* Tests that a useful exception is thrown when attempting to index a term that is
@@ -58,7 +59,7 @@ public class TestExceedMaxTermLength ext
(dir, newIndexWriterConfig(random(), new MockAnalyzer(random())));
try {
final FieldType ft = new FieldType();
- ft.setIndexed(true);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
ft.setStored(random().nextBoolean());
ft.freeze();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Oct 22 05:44:17 2014
@@ -1246,7 +1246,8 @@ public class TestIndexWriter extends Luc
customType.setTokenized(true);
Field f = new Field("binary", b, 10, 17, customType);
- customType.setIndexed(true);
+ // TODO: this is evil, changing the type after creating the field:
+ customType.setIndexOptions(IndexOptions.DOCS_ONLY);
final MockTokenizer doc1field1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
doc1field1.setReader(new StringReader("doc1field1"));
f.setTokenStream(doc1field1);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Wed Oct 22 05:44:17 2014
@@ -89,7 +89,7 @@ public class TestIndexWriterExceptions e
custom1.setStoreTermVectorOffsets(true);
custom2.setStored(true);
- custom2.setIndexed(true);
+ custom2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
custom3.setStored(true);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Wed Oct 22 05:44:17 2014
@@ -47,11 +47,6 @@ public class TestIndexableField extends
private final int counter;
private final IndexableFieldType fieldType = new IndexableFieldType() {
@Override
- public boolean indexed() {
- return (counter % 10) != 3;
- }
-
- @Override
public boolean stored() {
return (counter & 1) == 0 || (counter % 10) == 3;
}
@@ -63,7 +58,7 @@ public class TestIndexableField extends
@Override
public boolean storeTermVectors() {
- return indexed() && counter % 2 == 1 && counter % 10 != 9;
+ return indexOptions() != null && counter % 2 == 1 && counter % 10 != 9;
}
@Override
@@ -88,7 +83,7 @@ public class TestIndexableField extends
@Override
public FieldInfo.IndexOptions indexOptions() {
- return FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ return counter%10 == 3 ? null : FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
@Override
@@ -208,7 +203,7 @@ public class TestIndexableField extends
next = new MyField(finalBaseCount + (fieldUpto++-1));
}
- if (next != null && next.fieldType().indexed()) return true;
+ if (next != null && next.fieldType().indexOptions() != null) return true;
else return this.hasNext();
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Wed Oct 22 05:44:17 2014
@@ -362,7 +362,6 @@ public class TestPostingsOffsets extends
if (i < 99 && random().nextInt(2) == 0) {
// stored only
FieldType ft = new FieldType();
- ft.setIndexed(false);
ft.setStored(true);
doc.add(new Field("foo", "boo!", ft));
} else {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java Wed Oct 22 05:44:17 2014
@@ -176,7 +176,7 @@ public class TestSegmentReader extends L
// test omit norms
for (int i=0; i<DocHelper.fields.length; i++) {
IndexableField f = DocHelper.fields[i];
- if (f.fieldType().indexed()) {
+ if (f.fieldType().indexOptions() != null) {
assertEquals(reader.getNormValues(f.name()) != null, !f.fieldType().omitNorms());
assertEquals(reader.getNormValues(f.name()) != null, !DocHelper.noNorms.containsKey(f.name()));
if (reader.getNormValues(f.name()) == null) {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Oct 22 05:44:17 2014
@@ -31,6 +31,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TermQuery;
@@ -815,12 +816,12 @@ public class TestStressIndexing2 extends
case 0:
customType.setStored(true);
customType.setOmitNorms(true);
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.freeze();
fields.add(newField(fieldName, getString(1), customType));
break;
case 1:
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.setTokenized(true);
customType.freeze();
fields.add(newField(fieldName, getString(0), customType));
@@ -835,7 +836,7 @@ public class TestStressIndexing2 extends
break;
case 3:
customType.setStored(true);
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.setTokenized(true);
customType.freeze();
fields.add(newField(fieldName, getString(bigFieldSize), customType));
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java Wed Oct 22 05:44:17 2014
@@ -111,7 +111,7 @@ public class TestSimilarityBase extends
for (int i = 0; i < docs.length; i++) {
Document d = new Document();
FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setIndexed(false);
+ ft.setIndexOptions(null);
d.add(newField(FIELD_ID, Integer.toString(i), ft));
d.add(newTextField(FIELD_BODY, docs[i], Field.Store.YES));
writer.addDocument(d);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java Wed Oct 22 05:44:17 2014
@@ -76,7 +76,7 @@ public class TestDocIdSetBuilder extends
builder.or(new RoaringDocIdSet.Builder(maxDoc).add(doc).build().iterator());
}
for (int i = 0; i < numIterators; ++i) {
- final int baseInc = 2 + random().nextInt(10000);
+ final int baseInc = 2 + random().nextInt(10);
RoaringDocIdSet.Builder b = new RoaringDocIdSet.Builder(maxDoc);
for (int doc = random().nextInt(10000); doc < maxDoc; doc += baseInc + random().nextInt(2000)) {
b.add(doc);
Modified: lucene/dev/branches/lucene5969/lucene/demo/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/demo/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/demo/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/demo/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,9 +17,12 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="core-demo"/>
- <dependencies>
- <dependency org="javax.servlet" name="servlet-api" rev="${/javax.servlet/servlet-api}"/>
- <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
- </dependencies>
+ <info organisation="org.apache.lucene" module="core-demo"/>
+ <configurations defaultconfmapping="compile->master">
+ <conf name="compile" transitive="false"/>
+ </configurations>
+ <dependencies>
+ <dependency org="javax.servlet" name="servlet-api" rev="${/javax.servlet/servlet-api}" conf="compile"/>
+ <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
+ </dependencies>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/expressions/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/expressions/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/expressions/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/expressions/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,11 +17,14 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="expressions"/>
- <dependencies>
- <dependency org="org.antlr" name="antlr-runtime" rev="${/org.antlr/antlr-runtime}" transitive="false"/>
- <dependency org="org.ow2.asm" name="asm" rev="${/org.ow2.asm/asm}" transitive="false"/>
- <dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" transitive="false"/>
- <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
- </dependencies>
+ <info organisation="org.apache.lucene" module="expressions"/>
+ <configurations defaultconfmapping="compile->master">
+ <conf name="compile" transitive="false"/>
+ </configurations>
+ <dependencies>
+ <dependency org="org.antlr" name="antlr-runtime" rev="${/org.antlr/antlr-runtime}" conf="compile"/>
+ <dependency org="org.ow2.asm" name="asm" rev="${/org.ow2.asm/asm}" conf="compile"/>
+ <dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
+ <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
+ </dependencies>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/facet/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/facet/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/facet/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/facet/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="facet"/>
+ <info organisation="org.apache.lucene" module="facet"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java Wed Oct 22 05:44:17 2014
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
/**
* Add an instance of this to your {@link Document} for every facet label.
@@ -33,7 +34,7 @@ import org.apache.lucene.document.FieldT
public class FacetField extends Field {
static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java Wed Oct 22 05:44:17 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.facet.sortedse
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.facet.FacetField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
/** Add an instance of this to your Document for every facet
* label to be indexed via SortedSetDocValues. */
@@ -28,7 +29,7 @@ public class SortedSetDocValuesFacetFiel
/** Indexed {@link FieldType}. */
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java (original)
+++ lucene/dev/branches/lucene5969/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java Wed Oct 22 05:44:17 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.Facets;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BytesRef;
/** Add an instance of this to your {@link Document} to add
@@ -40,7 +41,7 @@ public class AssociationFacetField exten
/** Indexed {@link FieldType}. */
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene5969/lucene/grouping/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/grouping/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/grouping/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/grouping/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="grouping"/>
+ <info organisation="org.apache.lucene" module="grouping"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/highlighter/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/highlighter/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/highlighter/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/highlighter/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="highlighter"/>
+ <info organisation="org.apache.lucene" module="highlighter"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/join/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/join/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/join/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/join/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="join"/>
+ <info organisation="org.apache.lucene" module="join"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/memory/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/memory/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/memory/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/memory/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="memory"/>
+ <info organisation="org.apache.lucene" module="memory"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Wed Oct 22 05:44:17 2014
@@ -208,6 +208,8 @@ public class MemoryIndex {
private Counter bytesUsed;
private boolean frozen = false;
+
+ private Similarity normSimilarity = IndexSearcher.getDefaultSimilarity();
/**
* Sorts term entries into ascending order; also works for
@@ -445,7 +447,7 @@ public class MemoryIndex {
if (!fieldInfos.containsKey(fieldName)) {
fieldInfos.put(fieldName,
- new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
+ new FieldInfo(fieldName, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, -1, null));
}
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
@@ -500,6 +502,15 @@ public class MemoryIndex {
}
/**
+ * Set the Similarity to be used for calculating field norms
+ */
+ public void setSimilarity(Similarity similarity) {
+ if (frozen)
+ throw new IllegalArgumentException("Cannot set Similarity when MemoryIndex is frozen");
+ this.normSimilarity = similarity;
+ }
+
+ /**
* Creates and returns a searcher that can be used to execute arbitrary
* Lucene queries and to collect the resulting query results as hits.
*
@@ -508,7 +519,7 @@ public class MemoryIndex {
public IndexSearcher createSearcher() {
MemoryIndexReader reader = new MemoryIndexReader();
IndexSearcher searcher = new IndexSearcher(reader); // ensures no auto-close !!
- reader.setSearcher(searcher); // to later get hold of searcher.getSimilarity()
+ searcher.setSimilarity(normSimilarity);
return searcher;
}
@@ -524,6 +535,7 @@ public class MemoryIndex {
for (Map.Entry<String,Info> info : sortedFields) {
info.getValue().sortTerms();
}
+ calculateNormValues();
}
/**
@@ -744,8 +756,6 @@ public class MemoryIndex {
*/
private final class MemoryIndexReader extends LeafReader {
- private IndexSearcher searcher; // needed to find searcher.getSimilarity()
-
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
}
@@ -1169,15 +1179,6 @@ public class MemoryIndex {
return null;
}
}
-
- private Similarity getSimilarity() {
- if (searcher != null) return searcher.getSimilarity();
- return IndexSearcher.getDefaultSimilarity();
- }
-
- private void setSearcher(IndexSearcher searcher) {
- this.searcher = searcher;
- }
@Override
public int numDocs() {
@@ -1202,33 +1203,35 @@ public class MemoryIndex {
if (DEBUG) System.err.println("MemoryIndexReader.doClose");
}
- /** performance hack: cache norms to avoid repeated expensive calculations */
- private NumericDocValues cachedNormValues;
- private String cachedFieldName;
- private Similarity cachedSimilarity;
-
@Override
public NumericDocValues getNormValues(String field) {
- FieldInfo fieldInfo = fieldInfos.get(field);
- if (fieldInfo == null || fieldInfo.omitsNorms())
- return null;
- NumericDocValues norms = cachedNormValues;
- Similarity sim = getSimilarity();
- if (!field.equals(cachedFieldName) || sim != cachedSimilarity) { // not cached?
- Info info = getInfo(field);
- int numTokens = info != null ? info.numTokens : 0;
- int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
- float boost = info != null ? info.getBoost() : 1.0f;
- FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
- long value = sim.computeNorm(invertState);
- norms = new MemoryIndexNormDocValues(value);
- // cache it for future reuse
- cachedNormValues = norms;
- cachedFieldName = field;
- cachedSimilarity = sim;
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
- }
- return norms;
+ if (norms == null)
+ return calculateFieldNormValue(field);
+ return norms.get(field);
+ }
+
+ }
+
+ private Map<String, NumericDocValues> norms = null;
+
+ private NumericDocValues calculateFieldNormValue(String field) {
+ FieldInfo fieldInfo = fieldInfos.get(field);
+ if (fieldInfo == null)
+ return null;
+ Info info = fields.get(field);
+ int numTokens = info != null ? info.numTokens : 0;
+ int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
+ float boost = info != null ? info.getBoost() : 1.0f;
+ FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
+ long value = normSimilarity.computeNorm(invertState);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
+ return new MemoryIndexNormDocValues(value);
+ }
+
+ private void calculateNormValues() {
+ norms = new HashMap<>();
+ for (String field : fieldInfos.keySet()) {
+ norms.put(field, calculateFieldNormValue(field));
}
}
@@ -1239,6 +1242,8 @@ public class MemoryIndex {
this.fieldInfos.clear();
this.fields.clear();
this.sortedFields = null;
+ this.norms = null;
+ this.normSimilarity = IndexSearcher.getDefaultSimilarity();
byteBlockPool.reset(false, false); // no need to 0-fill the buffers
intBlockPool.reset(true, false); // here must must 0-fill since we use slices
this.frozen = false;
Modified: lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java Wed Oct 22 05:44:17 2014
@@ -18,13 +18,20 @@ package org.apache.lucene.index.memory;
*/
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.junit.internal.matchers.StringContains.containsString;
@@ -63,6 +70,14 @@ public class TestMemoryIndex extends Luc
assertThat(e.getMessage(), containsString("frozen"));
}
+ try {
+ mi.setSimilarity(new BM25Similarity(1, 1));
+ fail("Expected an IllegalArgumentException when setting the Similarity after calling freeze()");
+ }
+ catch (RuntimeException e) {
+ assertThat(e.getMessage(), containsString("frozen"));
+ }
+
assertThat(mi.search(new TermQuery(new Term("f1", "some"))), not(is(0.0f)));
mi.reset();
@@ -70,6 +85,32 @@ public class TestMemoryIndex extends Luc
assertThat(mi.search(new TermQuery(new Term("f1", "some"))), is(0.0f));
assertThat(mi.search(new TermQuery(new Term("f1", "wibble"))), not(is(0.0f)));
+ // check we can set the Similarity again
+ mi.setSimilarity(new DefaultSimilarity());
+
+ }
+
+ @Test
+ public void testSimilarities() throws IOException {
+
+ MemoryIndex mi = new MemoryIndex();
+ mi.addField("f1", "a long text field that contains many many terms", analyzer);
+
+ IndexSearcher searcher = mi.createSearcher();
+ LeafReader reader = (LeafReader) searcher.getIndexReader();
+ float n1 = reader.getNormValues("f1").get(0);
+
+ // Norms aren't cached, so we can change the Similarity
+ mi.setSimilarity(new DefaultSimilarity() {
+ @Override
+ public float lengthNorm(FieldInvertState state) {
+ return 74;
+ }
+ });
+ float n2 = reader.getNormValues("f1").get(0);
+
+ assertTrue(n1 != n2);
+
}
Modified: lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java (original)
+++ lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java Wed Oct 22 05:44:17 2014
@@ -432,7 +432,7 @@ public class TestMemoryIndexAgainstRAMDi
Document nextDoc = lineFileDocs.nextDoc();
Document doc = new Document();
for (Field field : nextDoc.getFields()) {
- if (field.fieldType().indexed()) {
+ if (field.fieldType().indexOptions() != null) {
doc.add(field);
if (random().nextInt(3) == 0) {
doc.add(field); // randomly add the same field twice
@@ -443,7 +443,7 @@ public class TestMemoryIndexAgainstRAMDi
writer.addDocument(doc);
writer.close();
for (IndexableField field : doc.indexableFields()) {
- memory.addField(field.name(), ((Field)field).stringValue(), mockAnalyzer);
+ memory.addField(field.name(), ((Field)field).stringValue(), mockAnalyzer);
}
DirectoryReader competitor = DirectoryReader.open(dir);
LeafReader memIndexReader= (LeafReader) memory.createSearcher().getIndexReader();
Modified: lucene/dev/branches/lucene5969/lucene/misc/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/misc/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/misc/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/misc/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="misc"/>
+ <info organisation="org.apache.lucene" module="misc"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Wed Oct 22 05:44:17 2014
@@ -212,7 +212,7 @@ public class UninvertingReader extends F
}
}
}
- filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
+ filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), type, -1, null));
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
Modified: lucene/dev/branches/lucene5969/lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java (original)
+++ lucene/dev/branches/lucene5969/lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java Wed Oct 22 05:44:17 2014
@@ -212,7 +212,6 @@ public class TestLazyDocument extends Lu
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
- ft.setIndexed(fieldInfo.isIndexed());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
doc.add(new Field(fieldInfo.name, value, ft));
Modified: lucene/dev/branches/lucene5969/lucene/queries/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/ivy.xml?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/ivy.xml (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/ivy.xml Wed Oct 22 05:44:17 2014
@@ -17,5 +17,5 @@
under the License.
-->
<ivy-module version="2.0">
- <info organisation="org.apache.lucene" module="queries"/>
+ <info organisation="org.apache.lucene" module="queries"/>
</ivy-module>
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -60,7 +60,14 @@ public abstract class DualFloatFunction
public float floatVal(int doc) {
return func(doc, aVals, bVals);
}
-
+ /**
+ * True if and only if <em>all</em> of the wrapped {@link FunctionValues}
+ * <code>exists</code> for the specified doc
+ */
+ @Override
+ public boolean exists(int doc) {
+ return MultiFunction.allExists(doc, aVals, bVals);
+ }
@Override
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java Wed Oct 22 05:44:17 2014
@@ -102,7 +102,7 @@ public class IfFunction extends BoolFunc
@Override
public boolean exists(int doc) {
- return true; // TODO: flow through to any sub-sources?
+ return ifVals.boolVal(doc) ? trueVals.exists(doc) : falseVals.exists(doc);
}
@Override
@@ -148,4 +148,4 @@ public class IfFunction extends BoolFunc
trueSource.createWeight(context, searcher);
falseSource.createWeight(context, searcher);
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -59,6 +59,10 @@ public class LinearFloatFunction extends
return vals.floatVal(doc) * slope + intercept;
}
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public String toString(int doc) {
return slope + "*float(" + vals.toString(doc) + ")+" + intercept;
}
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -35,11 +35,25 @@ public class MaxFloatFunction extends Mu
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- if (valsArr.length == 0) return 0.0f;
+ if ( ! exists(doc, valsArr) ) return 0.0f;
+
float val = Float.NEGATIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- val = Math.max(vals.floatVal(doc), val);
+ if (vals.exists(doc)) {
+ val = Math.max(vals.floatVal(doc), val);
+ }
}
return val;
}
+
+ /**
+ * True if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see MultiFunction#anyExists
+ */
+ @Override
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.anyExists(doc, valsArr);
+ }
}
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -35,11 +35,26 @@ public class MinFloatFunction extends Mu
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- if (valsArr.length == 0) return 0.0f;
+ if ( ! exists(doc, valsArr) ) return 0.0f;
+
float val = Float.POSITIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- val = Math.min(vals.floatVal(doc), val);
+ if (vals.exists(doc)) {
+ val = Math.min(vals.floatVal(doc), val);
+ }
}
return val;
}
+
+ /**
+ * True if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see MultiFunction#anyExists
+ */
+ @Override
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.anyExists(doc, valsArr);
+ }
+
}
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -40,6 +40,18 @@ public abstract class MultiFloatFunction
abstract protected String name();
abstract protected float func(int doc, FunctionValues[] valsArr);
+ /**
+ * Called by {@link FunctionValues#exists} for each document.
+ *
+ * Default impl returns true if <em>all</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see FunctionValues#exists
+ * @see MultiFunction#allExists
+ */
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.allExists(doc, valsArr);
+ }
@Override
public String description() {
@@ -70,21 +82,12 @@ public abstract class MultiFloatFunction
public float floatVal(int doc) {
return func(doc, valsArr);
}
- @Override
+ public boolean exists(int doc) {
+ return MultiFloatFunction.this.exists(doc, valsArr);
+ }
+ @Override
public String toString(int doc) {
- StringBuilder sb = new StringBuilder();
- sb.append(name()).append('(');
- boolean firstTime=true;
- for (FunctionValues vals : valsArr) {
- if (firstTime) {
- firstTime=false;
- } else {
- sb.append(',');
- }
- sb.append(vals.toString(doc));
- }
- sb.append(')');
- return sb.toString();
+ return MultiFunction.toString(name(), valsArr, doc);
}
};
}
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java Wed Oct 22 05:44:17 2014
@@ -43,6 +43,35 @@ public abstract class MultiFunction exte
return description(name(), sources);
}
+ /**
+ * Helper utility for {@link FunctionValues} wrapping multiple {@link FunctionValues}
+ *
+ * @return true if <em>all</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ */
+ public static boolean allExists(int doc, FunctionValues... values) {
+ for (FunctionValues v : values) {
+ if ( ! v.exists(doc) ) {
+ return false;
+ }
+ }
+ return true;
+ }
+ /**
+ * Helper utility for {@link FunctionValues} wrapping multiple {@link FunctionValues}
+ *
+ * @return true if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ */
+ public static boolean anyExists(int doc, FunctionValues... values) {
+ for (FunctionValues v : values) {
+ if ( v.exists(doc) ) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static String description(String name, List<ValueSource> sources) {
StringBuilder sb = new StringBuilder();
sb.append(name).append('(');
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -69,6 +69,10 @@ public class ReciprocalFloatFunction ext
return a/(m*vals.floatVal(doc) + b);
}
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public String toString(int doc) {
return Float.toString(a) + "/("
+ m + "*float(" + vals.toString(doc) + ')'
Modified: lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java?rev=1633538&r1=1633537&r2=1633538&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java (original)
+++ lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java Wed Oct 22 05:44:17 2014
@@ -70,21 +70,23 @@ public class ScaleFloatFunction extends
int maxDoc = leaf.reader().maxDoc();
FunctionValues vals = source.getValues(context, leaf);
for (int i=0; i<maxDoc; i++) {
-
- float val = vals.floatVal(i);
- if ((Float.floatToRawIntBits(val) & (0xff<<23)) == 0xff<<23) {
- // if the exponent in the float is all ones, then this is +Inf, -Inf or NaN
- // which don't make sense to factor into the scale function
- continue;
- }
- if (val < minVal) {
- minVal = val;
- }
- if (val > maxVal) {
- maxVal = val;
+ if ( ! vals.exists(i) ) {
+ continue;
+ }
+ float val = vals.floatVal(i);
+ if ((Float.floatToRawIntBits(val) & (0xff<<23)) == 0xff<<23) {
+ // if the exponent in the float is all ones, then this is +Inf, -Inf or NaN
+ // which don't make sense to factor into the scale function
+ continue;
+ }
+ if (val < minVal) {
+ minVal = val;
+ }
+ if (val > maxVal) {
+ maxVal = val;
+ }
}
}
- }
if (minVal == Float.POSITIVE_INFINITY) {
// must have been an empty index
@@ -114,6 +116,10 @@ public class ScaleFloatFunction extends
return new FloatDocValues(this) {
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public float floatVal(int doc) {
return (vals.floatVal(doc) - minSource) * scale + min;
}