You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/08/20 23:30:58 UTC
svn commit: r1515977 [3/4] - in /lucene/dev/trunk: ./ dev-tools/ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/codecs/src/java/org/apache/lucene/codecs/simp...
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Aug 20 21:30:55 2013
@@ -756,6 +756,11 @@ public class MemoryIndex {
return null;
}
+ @Override
+ public Bits getDocsWithField(String field) throws IOException {
+ return null;
+ }
+
private class MemoryFields extends Fields {
@Override
public Iterator<String> iterator() {
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java Tue Aug 20 21:30:55 2013
@@ -221,6 +221,27 @@ public class SortingAtomicReader extends
}
}
+ private static class SortingBits implements Bits {
+
+ private final Bits in;
+ private final Sorter.DocMap docMap;
+
+ public SortingBits(final Bits in, Sorter.DocMap docMap) {
+ this.in = in;
+ this.docMap = docMap;
+ }
+
+ @Override
+ public boolean get(int index) {
+ return in.get(docMap.newToOld(index));
+ }
+
+ @Override
+ public int length() {
+ return in.length();
+ }
+ }
+
private static class SortingSortedDocValues extends SortedDocValues {
private final SortedDocValues in;
@@ -743,20 +764,9 @@ public class SortingAtomicReader extends
final Bits inLiveDocs = in.getLiveDocs();
if (inLiveDocs == null) {
return null;
+ } else {
+ return new SortingBits(inLiveDocs, docMap);
}
- return new Bits() {
-
- @Override
- public boolean get(int index) {
- return inLiveDocs.get(docMap.newToOld(index));
- }
-
- @Override
- public int length() {
- return inLiveDocs.length();
- }
-
- };
}
@Override
@@ -797,6 +807,16 @@ public class SortingAtomicReader extends
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ Bits bits = in.getDocsWithField(field);
+ if (bits == null || bits instanceof Bits.MatchAllBits || bits instanceof Bits.MatchNoBits) {
+ return bits;
+ } else {
+ return new SortingBits(bits, docMap);
+ }
+ }
+
+ @Override
public Fields getTermVectors(final int docID) throws IOException {
return in.getTermVectors(docMap.newToOld(docID));
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Tue Aug 20 21:30:55 2013
@@ -45,12 +45,13 @@ public class BytesRefFieldSource extends
// To be sorted or not to be sorted, that is the question
// TODO: do it cleaner?
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ final Bits docsWithField = FieldCache.DEFAULT.getDocsWithField(readerContext.reader(), field);
final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
return new FunctionValues() {
@Override
public boolean exists(int doc) {
- return true; // doc values are dense
+ return docsWithField.get(doc);
}
@Override
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java Tue Aug 20 21:30:55 2013
@@ -68,7 +68,7 @@ public class DoubleFieldSource extends F
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -142,7 +142,7 @@ public class DoubleFieldSource extends F
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java Tue Aug 20 21:30:55 2013
@@ -72,7 +72,7 @@ public class FloatFieldSource extends Fi
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -88,7 +88,7 @@ public class FloatFieldSource extends Fi
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Tue Aug 20 21:30:55 2013
@@ -95,7 +95,7 @@ public class IntFieldSource extends Fiel
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -150,7 +150,7 @@ public class IntFieldSource extends Fiel
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Tue Aug 20 21:30:55 2013
@@ -81,7 +81,7 @@ public class LongFieldSource extends Fie
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -141,7 +141,7 @@ public class LongFieldSource extends Fie
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/trunk/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/site/xsl/index.xsl?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/trunk/lucene/site/xsl/index.xsl Tue Aug 20 21:30:55 2013
@@ -75,7 +75,7 @@
<li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
<li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
<li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
- <li><a href="core/org/apache/lucene/codecs/lucene42/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+ <li><a href="core/org/apache/lucene/codecs/lucene45/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
<li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
<li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
<li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java Tue Aug 20 21:30:55 2013
@@ -78,10 +78,12 @@ public class BBoxSimilarityValueSource e
@Override
public float floatVal(int doc) {
+ double minXVal = minX.get(doc);
+ double maxXVal = maxX.get(doc);
// make sure it has minX and area
- if (validMinX.get(doc) && validMaxX.get(doc)) {
+ if ((minXVal != 0 || validMinX.get(doc)) && (maxXVal != 0 || validMaxX.get(doc))) {
rect.reset(
- minX.get(doc), maxX.get(doc),
+ minXVal, maxXVal,
minY.get(doc), maxY.get(doc));
return (float) similarity.score(rect, null);
} else {
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Tue Aug 20 21:30:55 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -161,7 +161,7 @@ public class AnalyzingInfixSuggester ext
* codec to use. */
protected IndexWriterConfig getIndexWriterConfig(Version matchVersion, Analyzer indexAnalyzer) {
IndexWriterConfig iwc = new IndexWriterConfig(matchVersion, indexAnalyzer);
- iwc.setCodec(new Lucene42Codec());
+ iwc.setCodec(new Lucene45Codec());
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
return iwc;
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java Tue Aug 20 21:30:55 2013
@@ -23,10 +23,10 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
/**
- * Acts like {@link Lucene42Codec} but with additional asserts.
+ * Acts like {@link Lucene45Codec} but with additional asserts.
*/
public final class AssertingCodec extends FilterCodec {
@@ -37,7 +37,7 @@ public final class AssertingCodec extend
private final NormsFormat norms = new AssertingNormsFormat();
public AssertingCodec() {
- super("Asserting", new Lucene42Codec());
+ super("Asserting", new Lucene45Codec());
}
@Override
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Tue Aug 20 21:30:55 2013
@@ -24,7 +24,7 @@ import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat;
import org.apache.lucene.index.AssertingAtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
@@ -33,15 +33,16 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
/**
- * Just like {@link Lucene42DocValuesFormat} but with additional asserts.
+ * Just like {@link Lucene45DocValuesFormat} but with additional asserts.
*/
public class AssertingDocValuesFormat extends DocValuesFormat {
- private final DocValuesFormat in = new Lucene42DocValuesFormat();
+ private final DocValuesFormat in = new Lucene45DocValuesFormat();
public AssertingDocValuesFormat() {
super("Asserting");
@@ -75,11 +76,10 @@ public class AssertingDocValuesFormat ex
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
int count = 0;
for (Number v : values) {
- assert v != null;
count++;
}
assert count == maxDoc;
- checkIterator(values.iterator(), maxDoc);
+ checkIterator(values.iterator(), maxDoc, true);
in.addNumericField(field, values);
}
@@ -87,12 +87,11 @@ public class AssertingDocValuesFormat ex
public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
int count = 0;
for (BytesRef b : values) {
- assert b != null;
- assert b.isValid();
+ assert b == null || b.isValid();
count++;
}
assert count == maxDoc;
- checkIterator(values.iterator(), maxDoc);
+ checkIterator(values.iterator(), maxDoc, true);
in.addBinaryField(field, values);
}
@@ -117,15 +116,17 @@ public class AssertingDocValuesFormat ex
for (Number v : docToOrd) {
assert v != null;
int ord = v.intValue();
- assert ord >= 0 && ord < valueCount;
- seenOrds.set(ord);
+ assert ord >= -1 && ord < valueCount;
+ if (ord >= 0) {
+ seenOrds.set(ord);
+ }
count++;
}
assert count == maxDoc;
assert seenOrds.cardinality() == valueCount;
- checkIterator(values.iterator(), valueCount);
- checkIterator(docToOrd.iterator(), maxDoc);
+ checkIterator(values.iterator(), valueCount, false);
+ checkIterator(docToOrd.iterator(), maxDoc, false);
in.addSortedField(field, values, docToOrd);
}
@@ -169,38 +170,80 @@ public class AssertingDocValuesFormat ex
assert docCount == maxDoc;
assert seenOrds.cardinality() == valueCount;
- checkIterator(values.iterator(), valueCount);
- checkIterator(docToOrdCount.iterator(), maxDoc);
- checkIterator(ords.iterator(), ordCount);
+ checkIterator(values.iterator(), valueCount, false);
+ checkIterator(docToOrdCount.iterator(), maxDoc, false);
+ checkIterator(ords.iterator(), ordCount, false);
in.addSortedSetField(field, values, docToOrdCount, ords);
}
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+ }
+
+ static class AssertingNormsConsumer extends DocValuesConsumer {
+ private final DocValuesConsumer in;
+ private final int maxDoc;
+
+ AssertingNormsConsumer(DocValuesConsumer in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ }
- private <T> void checkIterator(Iterator<T> iterator, long expectedSize) {
- for (long i = 0; i < expectedSize; i++) {
- boolean hasNext = iterator.hasNext();
- assert hasNext;
- T v = iterator.next();
+ @Override
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ int count = 0;
+ for (Number v : values) {
assert v != null;
- try {
- iterator.remove();
- throw new AssertionError("broken iterator (supports remove): " + iterator);
- } catch (UnsupportedOperationException expected) {
- // ok
- }
- }
- assert !iterator.hasNext();
- try {
- iterator.next();
- throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
- } catch (NoSuchElementException expected) {
- // ok
+ count++;
}
+ assert count == maxDoc;
+ checkIterator(values.iterator(), maxDoc, false);
+ in.addNumericField(field, values);
}
-
+
@Override
public void close() throws IOException {
in.close();
}
+
+ @Override
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static <T> void checkIterator(Iterator<T> iterator, long expectedSize, boolean allowNull) {
+ for (long i = 0; i < expectedSize; i++) {
+ boolean hasNext = iterator.hasNext();
+ assert hasNext;
+ T v = iterator.next();
+ assert allowNull || v != null;
+ try {
+ iterator.remove();
+ throw new AssertionError("broken iterator (supports remove): " + iterator);
+ } catch (UnsupportedOperationException expected) {
+ // ok
+ }
+ }
+ assert !iterator.hasNext();
+ try {
+ iterator.next();
+ throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
+ } catch (NoSuchElementException expected) {
+ // ok
+ }
}
static class AssertingDocValuesProducer extends DocValuesProducer {
@@ -244,6 +287,15 @@ public class AssertingDocValuesFormat ex
assert values != null;
return new AssertingAtomicReader.AssertingSortedSetDocValues(values, maxDoc);
}
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ assert field.getDocValuesType() != null;
+ Bits bits = in.getDocsWithField(field);
+ assert bits != null;
+ assert bits.length() == maxDoc;
+ return new AssertingAtomicReader.AssertingBits(bits);
+ }
@Override
public void close() throws IOException {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Tue Aug 20 21:30:55 2013
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesConsumer;
+import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingNormsConsumer;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesProducer;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.index.SegmentReadState;
@@ -38,7 +38,7 @@ public class AssertingNormsFormat extend
public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
DocValuesConsumer consumer = in.normsConsumer(state);
assert consumer != null;
- return new AssertingDocValuesConsumer(consumer, state.segmentInfo.getDocCount());
+ return new AssertingNormsConsumer(consumer, state.segmentInfo.getDocCount());
}
@Override
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java Tue Aug 20 21:30:55 2013
@@ -23,10 +23,12 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
+import org.apache.lucene.codecs.diskdv.DiskNormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
/** Codec that tries to use as little ram as possible because he spent all his money on beer */
// TODO: better name :)
@@ -39,11 +41,11 @@ public class CheapBastardCodec extends F
private final StoredFieldsFormat storedFields = new Lucene40StoredFieldsFormat();
private final TermVectorsFormat termVectors = new Lucene40TermVectorsFormat();
// these go to disk for all docvalues/norms datastructures
- private final DocValuesFormat docValues = new CheapBastardDocValuesFormat();
- private final NormsFormat norms = new CheapBastardNormsFormat();
+ private final DocValuesFormat docValues = new DiskDocValuesFormat();
+ private final NormsFormat norms = new DiskNormsFormat();
public CheapBastardCodec() {
- super("CheapBastard", new Lucene42Codec());
+ super("CheapBastard", new Lucene45Codec());
}
public PostingsFormat postingsFormat() {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java Tue Aug 20 21:30:55 2013
@@ -23,13 +23,13 @@ import org.apache.lucene.codecs.FilterCo
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
/**
* A codec that uses {@link CompressingStoredFieldsFormat} for its stored
- * fields and delegates to {@link Lucene42Codec} for everything else.
+ * fields and delegates to {@link Lucene45Codec} for everything else.
*/
public abstract class CompressingCodec extends FilterCodec {
@@ -73,7 +73,7 @@ public abstract class CompressingCodec e
* Creates a compressing codec with a given segment suffix
*/
public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize) {
- super(name, new Lucene42Codec());
+ super(name, new Lucene45Codec());
this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize);
this.termVectorsFormat = new CompressingTermVectorsFormat(name, segmentSuffix, compressionMode, chunkSize);
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Tue Aug 20 21:30:55 2013
@@ -1,8 +1,6 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.util.packed.PackedInts;
@@ -42,9 +40,4 @@ public class FastCompressingCodec extend
public NormsFormat normsFormat() {
return new Lucene42NormsFormat(PackedInts.FAST);
}
-
- @Override
- public DocValuesFormat docValuesFormat() {
- return new Lucene42DocValuesFormat(PackedInts.FAST);
- }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Tue Aug 20 21:30:55 2013
@@ -1,8 +1,6 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.util.packed.PackedInts;
@@ -42,9 +40,4 @@ public class FastDecompressionCompressin
public NormsFormat normsFormat() {
return new Lucene42NormsFormat(PackedInts.DEFAULT);
}
-
- @Override
- public DocValuesFormat docValuesFormat() {
- return new Lucene42DocValuesFormat(PackedInts.DEFAULT);
- }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Tue Aug 20 21:30:55 2013
@@ -24,7 +24,9 @@ import java.util.TreeSet;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.MissingOrdRemapper;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -54,7 +56,7 @@ class Lucene40DocValuesWriter extends Do
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for (Number n : values) {
- long v = n.longValue();
+ long v = n == null ? 0 : n.longValue();
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
@@ -92,7 +94,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(1); // size
for (Number n : values) {
- output.writeByte(n.byteValue());
+ output.writeByte(n == null ? 0 : n.byteValue());
}
}
@@ -103,7 +105,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(2); // size
for (Number n : values) {
- output.writeShort(n.shortValue());
+ output.writeShort(n == null ? 0 : n.shortValue());
}
}
@@ -114,7 +116,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(4); // size
for (Number n : values) {
- output.writeInt(n.intValue());
+ output.writeInt(n == null ? 0 : n.intValue());
}
}
@@ -131,7 +133,7 @@ class Lucene40DocValuesWriter extends Do
// writes longs
output.writeByte(Lucene40DocValuesFormat.VAR_INTS_FIXED_64);
for (Number n : values) {
- output.writeLong(n.longValue());
+ output.writeLong(n == null ? 0 : n.longValue());
}
} else {
// writes packed ints
@@ -143,7 +145,8 @@ class Lucene40DocValuesWriter extends Do
PackedInts.bitsRequired(delta),
PackedInts.DEFAULT);
for (Number n : values) {
- writer.add(n.longValue() - minValue);
+ long v = n == null ? 0 : n.longValue();
+ writer.add(v - minValue);
}
writer.finish();
}
@@ -156,6 +159,9 @@ class Lucene40DocValuesWriter extends Do
int minLength = Integer.MAX_VALUE;
int maxLength = Integer.MIN_VALUE;
for (BytesRef b : values) {
+ if (b == null) {
+ b = new BytesRef(); // 4.0 doesnt distinguish
+ }
if (b.length > Lucene40DocValuesFormat.MAX_BINARY_FIELD_LENGTH) {
throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + Lucene40DocValuesFormat.MAX_BINARY_FIELD_LENGTH);
}
@@ -246,7 +252,9 @@ class Lucene40DocValuesWriter extends Do
output.writeInt(length);
for (BytesRef v : values) {
- output.writeBytes(v.bytes, v.offset, v.length);
+ if (v != null) {
+ output.writeBytes(v.bytes, v.offset, v.length);
+ }
}
}
@@ -267,7 +275,9 @@ class Lucene40DocValuesWriter extends Do
final long startPos = data.getFilePointer();
for (BytesRef v : values) {
- data.writeBytes(v.bytes, v.offset, v.length);
+ if (v != null) {
+ data.writeBytes(v.bytes, v.offset, v.length);
+ }
}
/* addresses */
@@ -282,7 +292,9 @@ class Lucene40DocValuesWriter extends Do
long currentPosition = 0;
for (BytesRef v : values) {
w.add(currentPosition);
- currentPosition += v.length;
+ if (v != null) {
+ currentPosition += v.length;
+ }
}
// write sentinel
assert currentPosition == maxAddress;
@@ -304,7 +316,7 @@ class Lucene40DocValuesWriter extends Do
// deduplicate
TreeSet<BytesRef> dictionary = new TreeSet<BytesRef>();
for (BytesRef v : values) {
- dictionary.add(BytesRef.deepCopyOf(v));
+ dictionary.add(v == null ? new BytesRef() : BytesRef.deepCopyOf(v));
}
/* values */
@@ -321,6 +333,9 @@ class Lucene40DocValuesWriter extends Do
final PackedInts.Writer w = PackedInts.getWriter(index, maxDoc, PackedInts.bitsRequired(valueCount-1), PackedInts.DEFAULT);
for (BytesRef v : values) {
+ if (v == null) {
+ v = new BytesRef();
+ }
int ord = dictionary.headSet(v).size();
w.add(ord);
}
@@ -341,7 +356,7 @@ class Lucene40DocValuesWriter extends Do
// deduplicate
TreeSet<BytesRef> dictionary = new TreeSet<BytesRef>();
for (BytesRef v : values) {
- dictionary.add(BytesRef.deepCopyOf(v));
+ dictionary.add(v == null ? new BytesRef() : BytesRef.deepCopyOf(v));
}
/* values */
@@ -362,7 +377,7 @@ class Lucene40DocValuesWriter extends Do
final PackedInts.Writer w = PackedInts.getWriter(index, maxDoc, PackedInts.bitsRequired(currentAddress), PackedInts.DEFAULT);
for (BytesRef v : values) {
- w.add(valueToAddress.get(v));
+ w.add(valueToAddress.get(v == null ? new BytesRef() : v));
}
w.finish();
}
@@ -388,6 +403,15 @@ class Lucene40DocValuesWriter extends Do
maxLength = Math.max(maxLength, b.length);
}
+ // but dont use fixed if there are missing values (we are simulating how lucene40 wrote dv...)
+ boolean anyMissing = false;
+ for (Number n : docToOrd) {
+ if (n.longValue() == -1) {
+ anyMissing = true;
+ break;
+ }
+ }
+
boolean success = false;
IndexOutput data = null;
IndexOutput index = null;
@@ -397,12 +421,22 @@ class Lucene40DocValuesWriter extends Do
try {
data = dir.createOutput(dataName, state.context);
index = dir.createOutput(indexName, state.context);
- if (minLength == maxLength) {
+ if (minLength == maxLength && !anyMissing) {
// fixed byte[]
addFixedSortedBytesField(field, data, index, values, docToOrd, minLength);
} else {
// var byte[]
- addVarSortedBytesField(field, data, index, values, docToOrd);
+ // three cases for simulating the old writer:
+ // 1. no missing
+ // 2. missing (and empty string in use): remap ord=-1 -> ord=0
+ // 3. missing (and empty string not in use): remap all ords +1, insert empty string into values
+ if (!anyMissing) {
+ addVarSortedBytesField(field, data, index, values, docToOrd);
+ } else if (minLength == 0) {
+ addVarSortedBytesField(field, data, index, values, MissingOrdRemapper.mapMissingToOrd0(docToOrd));
+ } else {
+ addVarSortedBytesField(field, data, index, MissingOrdRemapper.insertEmptyValue(values), MissingOrdRemapper.mapAllOrds(docToOrd));
+ }
}
success = true;
} finally {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Tue Aug 20 21:30:55 2013
@@ -438,14 +438,14 @@ public class AssertingAtomicReader exten
this.in = in;
this.maxDoc = maxDoc;
this.valueCount = in.getValueCount();
- assert valueCount >= 1 && valueCount <= maxDoc;
+ assert valueCount >= 0 && valueCount <= maxDoc;
}
@Override
public int getOrd(int docID) {
assert docID >= 0 && docID < maxDoc;
int ord = in.getOrd(docID);
- assert ord >= 0 && ord < valueCount;
+ assert ord >= -1 && ord < valueCount;
return ord;
}
@@ -607,6 +607,54 @@ public class AssertingAtomicReader exten
return null;
}
}
+
+ /** Wraps a Bits but with additional asserts */
+ public static class AssertingBits implements Bits {
+ final Bits in;
+
+ public AssertingBits(Bits in) {
+ this.in = in;
+ }
+
+ @Override
+ public boolean get(int index) {
+ assert index >= 0 && index < length();
+ return in.get(index);
+ }
+
+ @Override
+ public int length() {
+ return in.length();
+ }
+ }
+
+ @Override
+ public Bits getLiveDocs() {
+ Bits liveDocs = super.getLiveDocs();
+ if (liveDocs != null) {
+ assert maxDoc() == liveDocs.length();
+ liveDocs = new AssertingBits(liveDocs);
+ } else {
+ assert maxDoc() == numDocs();
+ assert !hasDeletions();
+ }
+ return liveDocs;
+ }
+
+ @Override
+ public Bits getDocsWithField(String field) throws IOException {
+ Bits docsWithField = super.getDocsWithField(field);
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ if (docsWithField != null) {
+ assert fi != null;
+ assert fi.hasDocValues();
+ assert maxDoc() == docsWithField.length();
+ docsWithField = new AssertingBits(docsWithField);
+ } else {
+ assert fi == null || fi.hasDocValues() == false;
+ }
+ return docsWithField;
+ }
// this is the same hack as FCInvisible
@Override
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Tue Aug 20 21:30:55 2013
@@ -26,6 +26,7 @@ import java.util.Map.Entry;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -51,6 +52,7 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.LuceneTestCase;
@@ -649,6 +651,43 @@ public abstract class BaseDocValuesForma
ireader.close();
directory.close();
}
+
+ public void testSortedMergeAwayAllValues() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField("field", new BytesRef("hello")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ iwriter.deleteDocuments(new Term("id", "1"));
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field");
+ if (defaultCodecSupportsDocsWithField()) {
+ assertEquals(-1, dv.getOrd(0));
+ assertEquals(0, dv.getValueCount());
+ } else {
+ assertEquals(0, dv.getOrd(0));
+ assertEquals(1, dv.getValueCount());
+ BytesRef ref = new BytesRef();
+ dv.lookupOrd(0, ref);
+ assertEquals(new BytesRef(), ref);
+ }
+
+ ireader.close();
+ directory.close();
+ }
public void testBytesWithNewline() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
@@ -695,7 +734,10 @@ public abstract class BaseDocValuesForma
BytesRef scratch = new BytesRef();
dv.lookupOrd(dv.getOrd(0), scratch);
assertEquals(new BytesRef("hello world 2"), scratch);
- dv.lookupOrd(dv.getOrd(1), scratch);
+ if (defaultCodecSupportsDocsWithField()) {
+ assertEquals(-1, dv.getOrd(1));
+ }
+ dv.get(1, scratch);
assertEquals(new BytesRef(""), scratch);
ireader.close();
directory.close();
@@ -1074,8 +1116,10 @@ public abstract class BaseDocValuesForma
doc.add(newTextField("id", "noValue", Field.Store.YES));
w.addDocument(doc);
}
- BytesRef bytesRef = new BytesRef();
- hash.add(bytesRef); // add empty value for the gaps
+ if (!defaultCodecSupportsDocsWithField()) {
+ BytesRef bytesRef = new BytesRef();
+ hash.add(bytesRef); // add empty value for the gaps
+ }
if (rarely()) {
w.commit();
}
@@ -1189,6 +1233,73 @@ public abstract class BaseDocValuesForma
dir.close();
}
+ private void doTestMissingVsFieldCache(final long minValue, final long maxValue) throws Exception {
+ doTestMissingVsFieldCache(new LongProducer() {
+ @Override
+ long next() {
+ return _TestUtil.nextLong(random(), minValue, maxValue);
+ }
+ });
+ }
+
+ private void doTestMissingVsFieldCache(LongProducer longs) throws Exception {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Field idField = new StringField("id", "", Field.Store.NO);
+ Field indexedField = newStringField("indexed", "", Field.Store.NO);
+ Field dvField = new NumericDocValuesField("dv", 0);
+
+
+ // index some docs
+ int numDocs = atLeast(300);
+ // numDocs should be always > 256 so that in case of a codec that optimizes
+ // for numbers of values <= 256, all storage layouts are tested
+ assert numDocs > 256;
+ for (int i = 0; i < numDocs; i++) {
+ idField.setStringValue(Integer.toString(i));
+ long value = longs.next();
+ indexedField.setStringValue(Long.toString(value));
+ dvField.setLongValue(value);
+ Document doc = new Document();
+ doc.add(idField);
+ // 1/4 of the time we neglect to add the fields
+ if (random().nextInt(4) > 0) {
+ doc.add(indexedField);
+ doc.add(dvField);
+ }
+ writer.addDocument(doc);
+ if (random().nextInt(31) == 0) {
+ writer.commit();
+ }
+ }
+
+ // delete some docs
+ int numDeletions = random().nextInt(numDocs/10);
+ for (int i = 0; i < numDeletions; i++) {
+ int id = random().nextInt(numDocs);
+ writer.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+
+ // merge some segments and ensure that at least one of them has more than
+ // 256 values
+ writer.forceMerge(numDocs / 256);
+
+ writer.close();
+
+ // compare
+ DirectoryReader ir = DirectoryReader.open(dir);
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader r = context.reader();
+ Bits expected = FieldCache.DEFAULT.getDocsWithField(r, "indexed");
+ Bits actual = FieldCache.DEFAULT.getDocsWithField(r, "dv");
+ assertEquals(expected, actual);
+ }
+ ir.close();
+ dir.close();
+ }
+
public void testBooleanNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1203,6 +1314,13 @@ public abstract class BaseDocValuesForma
}
}
+ public void testByteMissingVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestMissingVsFieldCache(Byte.MIN_VALUE, Byte.MAX_VALUE);
+ }
+ }
+
public void testShortNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1210,6 +1328,13 @@ public abstract class BaseDocValuesForma
}
}
+ public void testShortMissingVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestMissingVsFieldCache(Short.MIN_VALUE, Short.MAX_VALUE);
+ }
+ }
+
public void testIntNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1217,6 +1342,13 @@ public abstract class BaseDocValuesForma
}
}
+ public void testIntMissingVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestMissingVsFieldCache(Integer.MIN_VALUE, Integer.MAX_VALUE);
+ }
+ }
+
public void testLongNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1224,6 +1356,13 @@ public abstract class BaseDocValuesForma
}
}
+ public void testLongMissingVsFieldCache() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestMissingVsFieldCache(Long.MIN_VALUE, Long.MAX_VALUE);
+ }
+ }
+
private void doTestBinaryVsStoredFields(int minLength, int maxLength) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
@@ -1971,6 +2110,13 @@ public abstract class BaseDocValuesForma
doTestSortedSetVsStoredFields(1, 10);
}
}
+
+ private void assertEquals(Bits expected, Bits actual) throws Exception {
+ assertEquals(expected.length(), actual.length());
+ for (int i = 0; i < expected.length(); i++) {
+ assertEquals(expected.get(i), actual.get(i));
+ }
+ }
private void assertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) throws Exception {
assertEquals(maxDoc, new SingletonSortedSetDocValues(expected), new SingletonSortedSetDocValues(actual));
@@ -2198,6 +2344,206 @@ public abstract class BaseDocValuesForma
doTestNumericsVsStoredFields(longs);
}
}
+
+ public void testTwoNumbersOneMissing() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 0));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ NumericDocValues dv = ar.getNumericDocValues("dv1");
+ assertEquals(0, dv.get(0));
+ assertEquals(0, dv.get(1));
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ ir.close();
+ directory.close();
+ }
+
+ public void testTwoNumbersOneMissingWithMerging() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 0));
+ iw.addDocument(doc);
+ iw.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ NumericDocValues dv = ar.getNumericDocValues("dv1");
+ assertEquals(0, dv.get(0));
+ assertEquals(0, dv.get(1));
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ ir.close();
+ directory.close();
+ }
+
+ public void testThreeNumbersOneMissingWithMerging() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 0));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "2", Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 5));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ NumericDocValues dv = ar.getNumericDocValues("dv1");
+ assertEquals(0, dv.get(0));
+ assertEquals(0, dv.get(1));
+ assertEquals(5, dv.get(2));
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ assertTrue(docsWithField.get(2));
+ ir.close();
+ directory.close();
+ }
+
+ public void testTwoBytesOneMissing() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv1", new BytesRef()));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ BinaryDocValues dv = ar.getBinaryDocValues("dv1");
+ BytesRef ref = new BytesRef();
+ dv.get(0, ref);
+ assertEquals(new BytesRef(), ref);
+ dv.get(1, ref);
+ assertEquals(new BytesRef(), ref);
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ ir.close();
+ directory.close();
+ }
+
+ public void testTwoBytesOneMissingWithMerging() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv1", new BytesRef()));
+ iw.addDocument(doc);
+ iw.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ BinaryDocValues dv = ar.getBinaryDocValues("dv1");
+ BytesRef ref = new BytesRef();
+ dv.get(0, ref);
+ assertEquals(new BytesRef(), ref);
+ dv.get(1, ref);
+ assertEquals(new BytesRef(), ref);
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ ir.close();
+ directory.close();
+ }
+
+ public void testThreeBytesOneMissingWithMerging() throws IOException {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv1", new BytesRef()));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ iw.addDocument(doc);
+ iw.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "2", Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv1", new BytesRef("boo")));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+
+ IndexReader ir = DirectoryReader.open(directory);
+ assertEquals(1, ir.leaves().size());
+ AtomicReader ar = ir.leaves().get(0).reader();
+ BinaryDocValues dv = ar.getBinaryDocValues("dv1");
+ BytesRef ref = new BytesRef();
+ dv.get(0, ref);
+ assertEquals(new BytesRef(), ref);
+ dv.get(1, ref);
+ assertEquals(new BytesRef(), ref);
+ dv.get(2, ref);
+ assertEquals(new BytesRef("boo"), ref);
+ Bits docsWithField = ar.getDocsWithField("dv1");
+ assertTrue(docsWithField.get(0));
+ assertFalse(docsWithField.get(1));
+ assertTrue(docsWithField.get(2));
+ ir.close();
+ directory.close();
+ }
// LUCENE-4853
public void testHugeBinaryValues() throws Exception {
@@ -2295,6 +2641,7 @@ public abstract class BaseDocValuesForma
d.close();
}
+ // TODO: get this out of here and into the deprecated codecs (4.0, 4.2)
public void testHugeBinaryValueLimit() throws Exception {
// We only test DVFormats that have a limit
assumeFalse("test requires codec with limits on max binary field length", codecAcceptsHugeBinaryValues("field"));
@@ -2363,6 +2710,243 @@ public abstract class BaseDocValuesForma
ar.close();
d.close();
}
+
+ /** Tests dv against stored fields with threads (binary/numeric/sorted, no missing) */
+ public void testThreads() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ Field idField = new StringField("id", "", Field.Store.NO);
+ Field storedBinField = new StoredField("storedBin", new byte[0]);
+ Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
+ Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
+ Field storedNumericField = new StoredField("storedNum", "");
+ Field dvNumericField = new NumericDocValuesField("dvNum", 0);
+ doc.add(idField);
+ doc.add(storedBinField);
+ doc.add(dvBinField);
+ doc.add(dvSortedField);
+ doc.add(storedNumericField);
+ doc.add(dvNumericField);
+
+ // index some docs
+ int numDocs = atLeast(300);
+ for (int i = 0; i < numDocs; i++) {
+ idField.setStringValue(Integer.toString(i));
+ int length = _TestUtil.nextInt(random(), 0, 8);
+ byte buffer[] = new byte[length];
+ random().nextBytes(buffer);
+ storedBinField.setBytesValue(buffer);
+ dvBinField.setBytesValue(buffer);
+ dvSortedField.setBytesValue(buffer);
+ long numericValue = random().nextLong();
+ storedNumericField.setStringValue(Long.toString(numericValue));
+ dvNumericField.setLongValue(numericValue);
+ writer.addDocument(doc);
+ if (random().nextInt(31) == 0) {
+ writer.commit();
+ }
+ }
+
+ // delete some docs
+ int numDeletions = random().nextInt(numDocs/10);
+ for (int i = 0; i < numDeletions; i++) {
+ int id = random().nextInt(numDocs);
+ writer.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+ writer.close();
+
+ // compare
+ final DirectoryReader ir = DirectoryReader.open(dir);
+ int numThreads = _TestUtil.nextInt(random(), 2, 7);
+ Thread threads[] = new Thread[numThreads];
+ final CountDownLatch startingGun = new CountDownLatch(1);
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ startingGun.await();
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader r = context.reader();
+ BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
+ SortedDocValues sorted = r.getSortedDocValues("dvSorted");
+ NumericDocValues numerics = r.getNumericDocValues("dvNum");
+ for (int j = 0; j < r.maxDoc(); j++) {
+ BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
+ BytesRef scratch = new BytesRef();
+ binaries.get(j, scratch);
+ assertEquals(binaryValue, scratch);
+ sorted.get(j, scratch);
+ assertEquals(binaryValue, scratch);
+ String expected = r.document(j).get("storedNum");
+ assertEquals(Long.parseLong(expected), numerics.get(j));
+ }
+ }
+ _TestUtil.checkReader(ir);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].start();
+ }
+ startingGun.countDown();
+ for (Thread t : threads) {
+ t.join();
+ }
+ ir.close();
+ dir.close();
+ }
+
+ /** Tests dv against stored fields with threads (all types + missing) */
+ public void testThreads2() throws Exception {
+ assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
+ assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+ Field idField = new StringField("id", "", Field.Store.NO);
+ Field storedBinField = new StoredField("storedBin", new byte[0]);
+ Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
+ Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
+ Field storedNumericField = new StoredField("storedNum", "");
+ Field dvNumericField = new NumericDocValuesField("dvNum", 0);
+
+ // index some docs
+ int numDocs = atLeast(300);
+ for (int i = 0; i < numDocs; i++) {
+ idField.setStringValue(Integer.toString(i));
+ int length = _TestUtil.nextInt(random(), 0, 8);
+ byte buffer[] = new byte[length];
+ random().nextBytes(buffer);
+ storedBinField.setBytesValue(buffer);
+ dvBinField.setBytesValue(buffer);
+ dvSortedField.setBytesValue(buffer);
+ long numericValue = random().nextLong();
+ storedNumericField.setStringValue(Long.toString(numericValue));
+ dvNumericField.setLongValue(numericValue);
+ Document doc = new Document();
+ doc.add(idField);
+ if (random().nextInt(4) > 0) {
+ doc.add(storedBinField);
+ doc.add(dvBinField);
+ doc.add(dvSortedField);
+ }
+ if (random().nextInt(4) > 0) {
+ doc.add(storedNumericField);
+ doc.add(dvNumericField);
+ }
+ int numSortedSetFields = random().nextInt(3);
+ Set<String> values = new TreeSet<String>();
+ for (int j = 0; j < numSortedSetFields; j++) {
+ values.add(_TestUtil.randomSimpleString(random()));
+ }
+ for (String v : values) {
+ doc.add(new SortedSetDocValuesField("dvSortedSet", new BytesRef(v)));
+ doc.add(new StoredField("storedSortedSet", v));
+ }
+ writer.addDocument(doc);
+ if (random().nextInt(31) == 0) {
+ writer.commit();
+ }
+ }
+
+ // delete some docs
+ int numDeletions = random().nextInt(numDocs/10);
+ for (int i = 0; i < numDeletions; i++) {
+ int id = random().nextInt(numDocs);
+ writer.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+ writer.close();
+
+ // compare
+ final DirectoryReader ir = DirectoryReader.open(dir);
+ int numThreads = _TestUtil.nextInt(random(), 2, 7);
+ Thread threads[] = new Thread[numThreads];
+ final CountDownLatch startingGun = new CountDownLatch(1);
+
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ startingGun.await();
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader r = context.reader();
+ BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
+ Bits binaryBits = r.getDocsWithField("dvBin");
+ SortedDocValues sorted = r.getSortedDocValues("dvSorted");
+ Bits sortedBits = r.getDocsWithField("dvSorted");
+ NumericDocValues numerics = r.getNumericDocValues("dvNum");
+ Bits numericBits = r.getDocsWithField("dvNum");
+ SortedSetDocValues sortedSet = r.getSortedSetDocValues("dvSortedSet");
+ Bits sortedSetBits = r.getDocsWithField("dvSortedSet");
+ for (int j = 0; j < r.maxDoc(); j++) {
+ BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
+ if (binaryValue != null) {
+ if (binaries != null) {
+ BytesRef scratch = new BytesRef();
+ binaries.get(j, scratch);
+ assertEquals(binaryValue, scratch);
+ sorted.get(j, scratch);
+ assertEquals(binaryValue, scratch);
+ assertTrue(binaryBits.get(j));
+ assertTrue(sortedBits.get(j));
+ }
+ } else if (binaries != null) {
+ assertFalse(binaryBits.get(j));
+ assertFalse(sortedBits.get(j));
+ assertEquals(-1, sorted.getOrd(j));
+ }
+
+ String number = r.document(j).get("storedNum");
+ if (number != null) {
+ if (numerics != null) {
+ assertEquals(Long.parseLong(number), numerics.get(j));
+ }
+ } else if (numerics != null) {
+ assertFalse(numericBits.get(j));
+ assertEquals(0, numerics.get(j));
+ }
+
+ String values[] = r.document(j).getValues("storedSortedSet");
+ if (values.length > 0) {
+ assertNotNull(sortedSet);
+ sortedSet.setDocument(j);
+ for (int i = 0; i < values.length; i++) {
+ long ord = sortedSet.nextOrd();
+ assertTrue(ord != SortedSetDocValues.NO_MORE_ORDS);
+ BytesRef value = new BytesRef();
+ sortedSet.lookupOrd(ord, value);
+ assertEquals(values[i], value.utf8ToString());
+ }
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
+ assertTrue(sortedSetBits.get(j));
+ } else if (sortedSet != null) {
+ sortedSet.setDocument(j);
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
+ assertFalse(sortedSetBits.get(j));
+ }
+ }
+ }
+ _TestUtil.checkReader(ir);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].start();
+ }
+ startingGun.countDown();
+ for (Thread t : threads) {
+ t.join();
+ }
+ ir.close();
+ dir.close();
+ }
protected boolean codecAcceptsHugeBinaryValues(String field) {
return true;
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Tue Aug 20 21:30:55 2013
@@ -32,7 +32,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.compressing.CompressingCodec;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
@@ -502,7 +502,7 @@ public abstract class BaseStoredFieldsFo
// get another codec, other than the default: so we are merging segments across different codecs
final Codec otherCodec;
if ("SimpleText".equals(Codec.getDefault().getName())) {
- otherCodec = new Lucene42Codec();
+ otherCodec = new Lucene45Codec();
} else {
otherCodec = new SimpleTextCodec();
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java Tue Aug 20 21:30:55 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.Set;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FilterIterator;
/**
@@ -137,6 +138,11 @@ public final class FieldFilterAtomicRead
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ return hasField(field) ? super.getDocsWithField(field) : null;
+ }
+
+ @Override
public String toString() {
final StringBuilder sb = new StringBuilder("FieldFilterAtomicReader(reader=");
sb.append(in).append(", fields=");
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Tue Aug 20 21:30:55 2013
@@ -35,12 +35,12 @@ import org.apache.lucene.codecs.lucene41
import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapDocFreqInterval;
import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat;
import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
-import org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesFormat;
import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
+import org.apache.lucene.codecs.memory.MemoryDocValuesFormat;
import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
import org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat;
@@ -62,7 +62,7 @@ import org.apache.lucene.util._TestUtil;
* documents in different orders and the test will still be deterministic
* and reproducable.
*/
-public class RandomCodec extends Lucene42Codec {
+public class RandomCodec extends Lucene45Codec {
/** Shuffled list of postings formats to use for new mappings */
private List<PostingsFormat> formats = new ArrayList<PostingsFormat>();
@@ -148,11 +148,11 @@ public class RandomCodec extends Lucene4
new MemoryPostingsFormat(false, random.nextFloat()));
addDocValues(avoidCodecs,
- new Lucene42DocValuesFormat(),
+ new Lucene45DocValuesFormat(),
new DiskDocValuesFormat(),
+ new MemoryDocValuesFormat(),
new SimpleTextDocValuesFormat(),
- new AssertingDocValuesFormat(),
- new CheapBastardDocValuesFormat());
+ new AssertingDocValuesFormat());
Collections.shuffle(formats, random);
Collections.shuffle(dvFormats, random);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Aug 20 21:30:55 2013
@@ -1368,6 +1368,16 @@ public abstract class LuceneTestCase ext
}
return true;
}
+
+ /** Returns true if the codec "supports" docsWithField
+ * (other codecs return MatchAllBits, because you couldnt write missing values before) */
+ public static boolean defaultCodecSupportsDocsWithField() {
+ String name = Codec.getDefault().getName();
+ if (name.equals("Lucene40") || name.equals("Lucene41") || name.equals("Lucene42")) {
+ return false;
+ }
+ return true;
+ }
public void assertReaderEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException {
assertReaderStatisticsEquals(info, leftReader, rightReader);
@@ -1962,6 +1972,20 @@ public abstract class LuceneTestCase ext
assertNull(info, rightValues);
}
}
+
+ {
+ Bits leftBits = MultiDocValues.getDocsWithField(leftReader, field);
+ Bits rightBits = MultiDocValues.getDocsWithField(rightReader, field);
+ if (leftBits != null && rightBits != null) {
+ assertEquals(info, leftBits.length(), rightBits.length());
+ for (int i = 0; i < leftBits.length(); i++) {
+ assertEquals(info, leftBits.get(i), rightBits.get(i));
+ }
+ } else {
+ assertNull(info, leftBits);
+ assertNull(info, rightBits);
+ }
+ }
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Tue Aug 20 21:30:55 2013
@@ -40,6 +40,8 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.codecs.lucene40.Lucene40RWPostingsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene42.Lucene42RWCodec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.index.RandomCodec;
import org.apache.lucene.search.RandomSimilarityProvider;
@@ -161,6 +163,13 @@ final class TestRuleSetupAndRestoreClass
!shouldAvoidCodec("Lucene41"))) {
codec = Codec.forName("Lucene41");
assert codec instanceof Lucene41RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
+ } else if ("Lucene42".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
+ "random".equals(TEST_POSTINGSFORMAT) &&
+ "random".equals(TEST_DOCVALUESFORMAT) &&
+ randomVal == 2 &&
+ !shouldAvoidCodec("Lucene42"))) {
+ codec = Codec.forName("Lucene42");
+ assert codec instanceof Lucene42RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if (("random".equals(TEST_POSTINGSFORMAT) == false) || ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
// the user wired postings or DV: this is messy
// refactor into RandomCodec....
@@ -182,7 +191,7 @@ final class TestRuleSetupAndRestoreClass
dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
}
- codec = new Lucene42Codec() {
+ codec = new Lucene45Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return format;
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Tue Aug 20 21:30:55 2013
@@ -44,7 +44,7 @@ import java.util.zip.ZipFile;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -703,7 +703,7 @@ public class _TestUtil {
if (LuceneTestCase.VERBOSE) {
System.out.println("forcing postings format to:" + format);
}
- return new Lucene42Codec() {
+ return new Lucene45Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return format;
@@ -721,7 +721,7 @@ public class _TestUtil {
if (LuceneTestCase.VERBOSE) {
System.out.println("forcing docvalues format to:" + format);
}
- return new Lucene42Codec() {
+ return new Lucene45Codec() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
return format;
@@ -743,6 +743,7 @@ public class _TestUtil {
return p.getName();
}
}
+
public static String getDocValuesFormat(String field) {
return getDocValuesFormat(Codec.getDefault(), field);
}
@@ -756,11 +757,13 @@ public class _TestUtil {
}
}
+ // TODO: remove this, push this test to Lucene40/Lucene42 codec tests
public static boolean fieldSupportsHugeBinaryDocValues(String field) {
String dvFormat = getDocValuesFormat(field);
- return dvFormat.equals("CheapBastard") ||
- dvFormat.equals("Disk") ||
- dvFormat.equals("SimpleText");
+ if (dvFormat.equals("Lucene40") || dvFormat.equals("Lucene42") || dvFormat.equals("Memory")) {
+ return false;
+ }
+ return true;
}
public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {
Modified: lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Tue Aug 20 21:30:55 2013
@@ -21,3 +21,4 @@ org.apache.lucene.codecs.compressing.Hig
org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec
org.apache.lucene.codecs.lucene40.Lucene40RWCodec
org.apache.lucene.codecs.lucene41.Lucene41RWCodec
+org.apache.lucene.codecs.lucene42.Lucene42RWCodec
Modified: lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Tue Aug 20 21:30:55 2013
@@ -14,4 +14,4 @@
# limitations under the License.
org.apache.lucene.codecs.asserting.AssertingDocValuesFormat
-org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesFormat
\ No newline at end of file
+org.apache.lucene.codecs.lucene42.Lucene42RWDocValuesFormat
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Aug 20 21:30:55 2013
@@ -96,6 +96,10 @@ New Features
* SOLR-4718: Allow solr.xml to be stored in ZooKeeper
* SOLR-5156: Enhance ZkCLI to allow uploading of arbitrary files to ZK.
+* SOLR-5165: Single-valued docValues fields no longer require a default value.
+ Additionally they work with sortMissingFirst, sortMissingLast, facet.missing,
+ exists() in function queries, etc. (Robert Muir)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java Tue Aug 20 21:30:55 2013
@@ -3,7 +3,7 @@ package org.apache.solr.core;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -51,7 +51,7 @@ public class SchemaCodecFactory extends
@Override
public void init(NamedList args) {
super.init(args);
- codec = new Lucene42Codec() {
+ codec = new Lucene45Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
final SchemaField fieldOrNull = core.getLatestSchema().getFieldOrNull(field);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java Tue Aug 20 21:30:55 2013
@@ -218,11 +218,7 @@ public class DocValuesFacets {
static NamedList<Integer> finalize(NamedList<Integer> res, SolrIndexSearcher searcher, SchemaField schemaField, DocSet docs, int missingCount, boolean missing) throws IOException {
if (missing) {
if (missingCount < 0) {
- if (schemaField.multiValued()) {
- missingCount = SimpleFacets.getFieldMissingCount(searcher,docs,schemaField.getName());
- } else {
- missingCount = 0; // single-valued dv is implicitly 0
- }
+ missingCount = SimpleFacets.getFieldMissingCount(searcher,docs,schemaField.getName());
}
res.add(null, missingCount);
}
@@ -231,12 +227,12 @@ public class DocValuesFacets {
}
/** accumulates per-segment single-valued facet counts, mapping to global ordinal space */
- // specialized since the single-valued case is simpler: you don't have to deal with missing count, etc
+ // specialized since the single-valued case is different
static void accumSingle(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
int term = si.getOrd(doc);
- if (map != null) {
+ if (map != null && term >= 0) {
term = (int) map.getGlobalOrd(subIndex, term);
}
int arrIdx = term-startTermIndex;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java?rev=1515977&r1=1515976&r2=1515977&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java Tue Aug 20 21:30:55 2013
@@ -190,8 +190,9 @@ final class NumericFacets {
}
docsWithField = FieldCache.DEFAULT.getDocsWithField(ctx.reader(), fieldName);
}
- if (docsWithField.get(doc - ctx.docBase)) {
- hashTable.add(doc, longs.get(doc - ctx.docBase), 1);
+ long v = longs.get(doc - ctx.docBase);
+ if (v != 0 || docsWithField.get(doc - ctx.docBase)) {
+ hashTable.add(doc, v, 1);
} else {
++missingCount;
}
@@ -254,7 +255,7 @@ final class NumericFacets {
if (zeros && (limit < 0 || result.size() < limit)) { // need to merge with the term dict
if (!sf.indexed()) {
- throw new IllegalStateException("Cannot use " + FacetParams.FACET_MINCOUNT + "=0 on a field which is not indexed");
+ throw new IllegalStateException("Cannot use " + FacetParams.FACET_MINCOUNT + "=0 on field " + sf.getName() + " which is not indexed");
}
// Add zeros until there are limit results
final Set<String> alreadySeen = new HashSet<String>();