You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/09/21 13:42:05 UTC
[01/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Repository: lucene-solr
Updated Branches:
refs/heads/master 93bc88d96 -> f7aa200d4
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java
index ee27f6e..2d2c381 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java
@@ -19,8 +19,10 @@ package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -58,6 +60,8 @@ import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestFieldCache extends LuceneTestCase {
private static LeafReader reader;
private static int NUM_DOCS;
@@ -132,28 +136,28 @@ public class TestFieldCache extends LuceneTestCase {
public void test() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
- NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", doubles, cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER, random().nextBoolean()));
+ NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.get(i));
+ assertEquals(i, doubles.nextDoc());
+ assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.longValue());
}
- NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LONG_POINT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", longs, cache.getNumerics(reader, "theLong", FieldCache.LONG_POINT_PARSER, random().nextBoolean()));
+ NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LONG_POINT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Long.MAX_VALUE - i, longs.get(i));
+ assertEquals(i, longs.nextDoc());
+ assertEquals(Long.MAX_VALUE - i, longs.longValue());
}
- NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.INT_POINT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", ints, cache.getNumerics(reader, "theInt", FieldCache.INT_POINT_PARSER, random().nextBoolean()));
+ NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.INT_POINT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Integer.MAX_VALUE - i, ints.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(Integer.MAX_VALUE - i, ints.longValue());
}
- NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.FLOAT_POINT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", floats, cache.getNumerics(reader, "theFloat", FieldCache.FLOAT_POINT_PARSER, random().nextBoolean()));
+ NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.FLOAT_POINT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.get(i));
+ assertEquals(i, floats.nextDoc());
+ assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.longValue());
}
Bits docsWithField = cache.getDocsWithField(reader, "theLong", FieldCache.LONG_POINT_PARSER);
@@ -176,11 +180,13 @@ public class TestFieldCache extends LuceneTestCase {
SortedDocValues termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
for (int i = 0; i < NUM_DOCS; i++) {
final String s;
- final int ord = termsIndex.getOrd(i);
- if (ord == -1) {
- s = null;
+ if (i > termsIndex.docID()) {
+ termsIndex.advance(i);
+ }
+ if (i == termsIndex.docID()) {
+ s = termsIndex.binaryValue().utf8ToString();
} else {
- s = termsIndex.lookupOrd(ord).utf8ToString();
+ s = null;
}
assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
}
@@ -214,20 +220,20 @@ public class TestFieldCache extends LuceneTestCase {
termsIndex = cache.getTermsIndex(reader, "bogusfield");
// getTerms
- BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString", true);
- Bits bits = cache.getDocsWithField(reader, "theRandomUnicodeString", null);
+ BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString");
for (int i = 0; i < NUM_DOCS; i++) {
- final String s;
- if (!bits.get(i)) {
- s = null;
+ if (terms.docID() < i) {
+ terms.nextDoc();
+ }
+ if (terms.docID() == i) {
+ assertEquals(unicodeStrings[i], terms.binaryValue().utf8ToString());
} else {
- s = terms.get(i).utf8ToString();
+ assertNull(unicodeStrings[i]);
}
- assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
}
// test bad field
- terms = cache.getTerms(reader, "bogusfield", false);
+ terms = cache.getTerms(reader, "bogusfield");
// getDocTermOrds
SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField", null);
@@ -237,7 +243,6 @@ public class TestFieldCache extends LuceneTestCase {
assertEquals(numEntries, cache.getCacheEntries().length);
for (int i = 0; i < NUM_DOCS; i++) {
- termOrds.setDocument(i);
// This will remove identical terms. A DocTermOrds doesn't return duplicate ords for a docId
List<BytesRef> values = new ArrayList<>(new LinkedHashSet<>(Arrays.asList(multiValued[i])));
for (BytesRef v : values) {
@@ -245,12 +250,17 @@ public class TestFieldCache extends LuceneTestCase {
// why does this test use null values... instead of an empty list: confusing
break;
}
+ if (i > termOrds.docID()) {
+ assertEquals(i, termOrds.nextDoc());
+ }
long ord = termOrds.nextOrd();
assert ord != SortedSetDocValues.NO_MORE_ORDS;
BytesRef scratch = termOrds.lookupOrd(ord);
assertEquals(v, scratch);
}
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd());
+ if (i == termOrds.docID()) {
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd());
+ }
}
// test bad field
@@ -267,7 +277,7 @@ public class TestFieldCache extends LuceneTestCase {
IndexReader r = DirectoryReader.open(dir);
LeafReader reader = SlowCompositeReaderWrapper.wrap(r);
TestUtil.checkReader(reader);
- FieldCache.DEFAULT.getTerms(reader, "foobar", true);
+ FieldCache.DEFAULT.getTerms(reader, "foobar");
FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
r.close();
@@ -294,7 +304,7 @@ public class TestFieldCache extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER, true);
+ cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER);
// The double[] takes one slots, and docsWithField should also
// have been populated:
@@ -305,27 +315,20 @@ public class TestFieldCache extends LuceneTestCase {
assertEquals(2, cache.getCacheEntries().length);
assertTrue(bits instanceof Bits.MatchAllBits);
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER, true);
- assertEquals(4, cache.getCacheEntries().length);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER);
+ NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER);
assertEquals(4, cache.getCacheEntries().length);
- for (int i = 0; i < docsWithField.length(); i++) {
+ for (int i = 0; i < reader.maxDoc(); i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(i, ints.longValue());
}
}
- NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.INT_POINT_PARSER, random().nextBoolean());
- docsWithField = cache.getDocsWithField(reader, "numInt", FieldCache.INT_POINT_PARSER);
- for (int i = 0; i < docsWithField.length(); i++) {
+ NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.INT_POINT_PARSER);
+ for (int i = 0; i < reader.maxDoc(); i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, numInts.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, numInts.nextDoc());
+ assertEquals(i, numInts.longValue());
}
}
}
@@ -368,14 +371,11 @@ public class TestFieldCache extends LuceneTestCase {
assertEquals(i%2 == 0, docsWithField.get(i));
}
} else {
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER, true);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER);
- for (int i = 0; i < docsWithField.length(); i++) {
+ NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER);
+ for (int i = 0; i < reader.maxDoc();i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(i, ints.longValue());
}
}
}
@@ -413,11 +413,12 @@ public class TestFieldCache extends LuceneTestCase {
// Binary type: can be retrieved via getTerms()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER);
});
- BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
- final BytesRef term = binary.get(0);
+ BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary");
+ assertEquals(0, binary.nextDoc());
+ final BytesRef term = binary.binaryValue();
assertEquals("binary value", term.utf8ToString());
expectThrows(IllegalStateException.class, () -> {
@@ -437,25 +438,28 @@ public class TestFieldCache extends LuceneTestCase {
// Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER);
});
expectThrows(IllegalStateException.class, () -> {
new DocTermOrds(ar, null, "sorted");
});
- binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
- BytesRef scratch = binary.get(0);
+ binary = FieldCache.DEFAULT.getTerms(ar, "sorted");
+ assertEquals(0, binary.nextDoc());
+
+ BytesRef scratch = binary.binaryValue();
assertEquals("sorted value", scratch.utf8ToString());
SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
- assertEquals(0, sorted.getOrd(0));
+ assertEquals(0, sorted.nextDoc());
+ assertEquals(0, sorted.ordValue());
assertEquals(1, sorted.getValueCount());
- scratch = sorted.get(0);
+ scratch = sorted.binaryValue();
assertEquals("sorted value", scratch.utf8ToString());
SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null);
- sortedSet.setDocument(0);
+ assertEquals(0, sortedSet.nextDoc());
assertEquals(0, sortedSet.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
assertEquals(1, sortedSet.getValueCount());
@@ -464,11 +468,12 @@ public class TestFieldCache extends LuceneTestCase {
assertTrue(bits.get(0));
// Numeric type: can be retrieved via getInts() and so on
- NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER, false);
- assertEquals(42, numeric.get(0));
+ NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER);
+ assertEquals(0, numeric.nextDoc());
+ assertEquals(42, numeric.longValue());
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getTerms(ar, "numeric", true);
+ FieldCache.DEFAULT.getTerms(ar, "numeric");
});
expectThrows(IllegalStateException.class, () -> {
@@ -488,11 +493,11 @@ public class TestFieldCache extends LuceneTestCase {
// SortedSet type: can be retrieved via getDocTermOrds()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER);
});
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getTerms(ar, "sortedset", true);
+ FieldCache.DEFAULT.getTerms(ar, "sortedset");
});
expectThrows(IllegalStateException.class, () -> {
@@ -504,7 +509,7 @@ public class TestFieldCache extends LuceneTestCase {
});
sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset", null);
- sortedSet.setDocument(0);
+ assertEquals(0, sortedSet.nextDoc());
assertEquals(0, sortedSet.nextOrd());
assertEquals(1, sortedSet.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
@@ -531,30 +536,26 @@ public class TestFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER, true);
- assertEquals(0, ints.get(0));
+ NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER, true);
- assertEquals(0, longs.get(0));
+ NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER, true);
- assertEquals(0, floats.get(0));
+ NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, floats.nextDoc());
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER, true);
- assertEquals(0, doubles.get(0));
+ NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, doubles.nextDoc());
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
- BytesRef scratch = binaries.get(0);
- assertEquals(0, scratch.length);
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ assertEquals(NO_MORE_DOCS, binaries.nextDoc());
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
- assertEquals(-1, sorted.getOrd(0));
- scratch = sorted.get(0);
- assertEquals(0, scratch.length);
+ assertEquals(NO_MORE_DOCS, sorted.nextDoc());
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);
- sortedSet.setDocument(0);
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
+ assertEquals(NO_MORE_DOCS, sortedSet.nextDoc());
Bits bits = cache.getDocsWithField(ar, "bogusbits", null);
assertFalse(bits.get(0));
@@ -589,30 +590,26 @@ public class TestFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER, true);
- assertEquals(0, ints.get(0));
+ NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER, true);
- assertEquals(0, longs.get(0));
+ NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER, true);
- assertEquals(0, floats.get(0));
+ NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, floats.nextDoc());
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER, true);
- assertEquals(0, doubles.get(0));
+ NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER);
+ assertEquals(NO_MORE_DOCS, doubles.nextDoc());
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
- BytesRef scratch = binaries.get(0);
- assertEquals(0, scratch.length);
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ assertEquals(NO_MORE_DOCS, binaries.nextDoc());
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
- assertEquals(-1, sorted.getOrd(0));
- scratch = sorted.get(0);
- assertEquals(0, scratch.length);
+ assertEquals(NO_MORE_DOCS, sorted.nextDoc());
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);
- sortedSet.setDocument(0);
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
+ assertEquals(NO_MORE_DOCS, sortedSet.nextDoc());
Bits bits = cache.getDocsWithField(ar, "bogusbits", null);
assertFalse(bits.get(0));
@@ -635,6 +632,7 @@ public class TestFieldCache extends LuceneTestCase {
doc.add(field);
doc.add(field2);
final long[] values = new long[TestUtil.nextInt(random(), 1, 10)];
+ Set<Integer> missing = new HashSet<>();
for (int i = 0; i < values.length; ++i) {
final long v;
switch (random().nextInt(10)) {
@@ -655,6 +653,7 @@ public class TestFieldCache extends LuceneTestCase {
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
+ missing.add(i);
} else {
field.setLongValue(v);
field2.setLongValue(v);
@@ -663,10 +662,14 @@ public class TestFieldCache extends LuceneTestCase {
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
- final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LONG_POINT_PARSER, false);
+ final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LONG_POINT_PARSER);
for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], longs.get(i));
+ if (missing.contains(i) == false) {
+ assertEquals(i, longs.nextDoc());
+ assertEquals(values[i], longs.longValue());
+ }
}
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
reader.close();
iw.close();
dir.close();
@@ -682,6 +685,7 @@ public class TestFieldCache extends LuceneTestCase {
IntPoint field = new IntPoint("f", 0);
doc.add(field);
final int[] values = new int[TestUtil.nextInt(random(), 1, 10)];
+ Set<Integer> missing = new HashSet<>();
for (int i = 0; i < values.length; ++i) {
final int v;
switch (random().nextInt(10)) {
@@ -702,6 +706,7 @@ public class TestFieldCache extends LuceneTestCase {
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
+ missing.add(i);
} else {
field.setIntValue(v);
iw.addDocument(doc);
@@ -709,10 +714,14 @@ public class TestFieldCache extends LuceneTestCase {
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
- final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.INT_POINT_PARSER, false);
+ final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.INT_POINT_PARSER);
for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], ints.get(i));
+ if (missing.contains(i) == false) {
+ assertEquals(i, ints.nextDoc());
+ assertEquals(values[i], ints.longValue());
+ }
}
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
reader.close();
iw.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java
index 18c6420..410f920 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java
@@ -48,8 +48,9 @@ public class TestFieldCacheReopen extends LuceneTestCase {
// Open reader1
DirectoryReader r = DirectoryReader.open(dir);
LeafReader r1 = getOnlyLeafReader(r);
- final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER, false);
- assertEquals(17, ints.get(0));
+ final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER);
+ assertEquals(0, ints.nextDoc());
+ assertEquals(17, ints.longValue());
// Add new segment
writer.addDocument(doc);
@@ -60,9 +61,10 @@ public class TestFieldCacheReopen extends LuceneTestCase {
assertNotNull(r2);
r.close();
LeafReader sub0 = r2.leaves().get(0).reader();
- final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.INT_POINT_PARSER, false);
+ final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.INT_POINT_PARSER);
r2.close();
- assertTrue(ints == ints2);
+ assertEquals(0, ints2.nextDoc());
+ assertEquals(17, ints2.longValue());
writer.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java
index a5958f9..b031681 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java
@@ -94,11 +94,11 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getNumerics(readerA, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false);
- cache.getNumerics(readerAclone, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false);
- cache.getNumerics(readerB, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false);
+ cache.getNumerics(readerA, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
+ cache.getNumerics(readerAclone, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
+ cache.getNumerics(readerB, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
- cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER, false);
+ cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER);
// // //
@@ -117,8 +117,8 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER, false);
- cache.getTerms(readerX, "theInt", false);
+ cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER);
+ cache.getTerms(readerX, "theInt");
// // //
@@ -140,9 +140,9 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getTerms(readerA, "theInt", false);
- cache.getTerms(readerB, "theInt", false);
- cache.getTerms(readerX, "theInt", false);
+ cache.getTerms(readerA, "theInt");
+ cache.getTerms(readerB, "theInt");
+ cache.getTerms(readerX, "theInt");
// // //
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java
index dedb449..bc04a6f 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java
@@ -281,7 +281,8 @@ public class TestFieldCacheSortRandom extends LuceneTestCase {
if (random.nextFloat() <= density) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
- matchValues.add(docValues.get((int) idSource.get(docID)));
+ assertEquals(docID, idSource.advance(docID));
+ matchValues.add(docValues.get((int) idSource.longValue()));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
index 0f7a13f..308a870 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
@@ -31,7 +31,6 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
@@ -51,6 +50,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
public class TestFieldCacheVsDocValues extends LuceneTestCase {
@@ -200,10 +200,11 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
TestUtil.checkReader(ar);
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field");
for(int docID=0;docID<docBytes.size();docID++) {
Document doc = ar.document(docID);
- BytesRef bytes = s.get(docID);
+ assertEquals(docID, s.nextDoc());
+ BytesRef bytes = s.binaryValue();
byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
assertEquals(expected.length, bytes.length);
assertEquals(new BytesRef(expected), bytes);
@@ -272,13 +273,13 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
w.close();
LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(ar
- );
+ TestUtil.checkReader(ar);
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field");
for(int docID=0;docID<docBytes.size();docID++) {
+ assertEquals(docID, s.nextDoc());
Document doc = ar.document(docID);
- BytesRef bytes = s.get(docID);
+ BytesRef bytes = s.binaryValue();
byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
assertEquals(expected.length, bytes.length);
assertEquals(new BytesRef(expected), bytes);
@@ -487,7 +488,35 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
}
private void assertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) throws Exception {
- assertEquals(maxDoc, DocValues.singleton(expected), DocValues.singleton(actual));
+ // can be null for the segment if no docs actually had any SortedDocValues
+ // in this case FC.getDocTermsOrds returns EMPTY
+ if (actual == null) {
+ assertEquals(expected.getValueCount(), 0);
+ return;
+ }
+ assertEquals(expected.getValueCount(), actual.getValueCount());
+
+ // compare ord lists
+ while (true) {
+ int docID = expected.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ assertEquals(NO_MORE_DOCS, actual.nextDoc());
+ break;
+ }
+ assertEquals(docID, actual.nextDoc());
+ assertEquals(expected.ordValue(), actual.ordValue());
+ assertEquals(expected.binaryValue(), actual.binaryValue());
+ }
+
+ // compare ord dictionary
+ for (long i = 0; i < expected.getValueCount(); i++) {
+ final BytesRef expectedBytes = BytesRef.deepCopyOf(expected.lookupOrd((int) i));
+ final BytesRef actualBytes = actual.lookupOrd((int) i);
+ assertEquals(expectedBytes, actualBytes);
+ }
+
+ // compare termsenum
+ assertEquals(expected.getValueCount(), expected.termsEnum(), actual.termsEnum());
}
private void assertEquals(int maxDoc, SortedSetDocValues expected, SortedSetDocValues actual) throws Exception {
@@ -498,10 +527,12 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
return;
}
assertEquals(expected.getValueCount(), actual.getValueCount());
- // compare ord lists
- for (int i = 0; i < maxDoc; i++) {
- expected.setDocument(i);
- actual.setDocument(i);
+ while (true) {
+ int docID = expected.nextDoc();
+ assertEquals(docID, actual.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
long expectedOrd;
while ((expectedOrd = expected.nextOrd()) != NO_MORE_ORDS) {
assertEquals(expectedOrd, actual.nextOrd());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java
index b0801fe..2caa1a1 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java
@@ -84,33 +84,46 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
@Override
public void run() {
try {
- //NumericDocValues ndv = ar.getNumericDocValues("number");
- NumericDocValues ndv = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false);
- //BinaryDocValues bdv = ar.getBinaryDocValues("bytes");
- BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes", false);
- SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
startingGun.await();
int iters = atLeast(1000);
for(int iter=0;iter<iters;iter++) {
int docID = threadRandom.nextInt(numDocs);
switch(threadRandom.nextInt(4)) {
case 0:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.INT_POINT_PARSER, false).get(docID));
+ {
+ NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.INT_POINT_PARSER);
+ assertEquals(docID, values.advance(docID));
+ assertEquals(numbers.get(docID).longValue(), values.longValue());
+ }
break;
case 1:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false).get(docID));
+ {
+ NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER);
+ assertEquals(docID, values.advance(docID));
+ assertEquals(numbers.get(docID).longValue(), values.longValue());
+ }
break;
case 2:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.FLOAT_POINT_PARSER, false).get(docID));
+ {
+ NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.FLOAT_POINT_PARSER);
+ assertEquals(docID, values.advance(docID));
+ assertEquals(numbers.get(docID).longValue(), values.longValue());
+ }
break;
case 3:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.DOUBLE_POINT_PARSER, false).get(docID));
+ {
+ NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.DOUBLE_POINT_PARSER);
+ assertEquals(docID, values.advance(docID));
+ assertEquals(numbers.get(docID).longValue(), values.longValue());
+ }
break;
}
- BytesRef term = bdv.get(docID);
- assertEquals(binary.get(docID), term);
- term = sdv.get(docID);
- assertEquals(sorted.get(docID), term);
+ BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes");
+ assertEquals(docID, bdv.advance(docID));
+ assertEquals(binary.get(docID), bdv.binaryValue());
+ SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
+ assertEquals(docID, sdv.advance(docID));
+ assertEquals(sorted.get(docID), sdv.binaryValue());
}
} catch (Exception e) {
throw new RuntimeException(e);
@@ -202,14 +215,25 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
+ int[] docIDToIDArray = new int[sr.maxDoc()];
+ for(int i=0;i<sr.maxDoc();i++) {
+ try {
+ assertEquals(i, docIDToID.nextDoc());
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ docIDToIDArray[i] = (int) docIDToID.longValue();
+ }
while(System.nanoTime() < END_TIME) {
- final SortedDocValues source;
- source = stringDVDirect;
-
for(int iter=0;iter<100;iter++) {
final int docID = random.nextInt(sr.maxDoc());
- BytesRef term = source.get(docID);
- assertEquals(docValues.get((int) docIDToID.get(docID)), term);
+ try {
+ SortedDocValues dvs = sr.getSortedDocValues("stringdv");
+ assertEquals(docID, dvs.advance(docID));
+ assertEquals(docValues.get(docIDToIDArray[docID]), dvs.binaryValue());
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java
index 5220460..9dc047b 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java
@@ -16,13 +16,6 @@
*/
package org.apache.solr.uninverting;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@@ -56,6 +49,17 @@ import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/** random assortment of tests against legacy numerics */
public class TestLegacyFieldCache extends LuceneTestCase {
private static LeafReader reader;
@@ -108,7 +112,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
cache.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
- cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false);
+ cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
cache.getNumerics(reader, "theDouble", new FieldCache.Parser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
@@ -120,7 +124,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
if (val<0) val ^= 0x7fffffff;
return val;
}
- }, false);
+ });
assertTrue(bos.toString(IOUtils.UTF_8).indexOf("WARNING") != -1);
} finally {
FieldCache.DEFAULT.setInfoStream(null);
@@ -130,28 +134,28 @@ public class TestLegacyFieldCache extends LuceneTestCase {
public void test() throws IOException {
FieldCache cache = FieldCache.DEFAULT;
- NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", doubles, cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean()));
+ NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.get(i));
+ assertEquals(i, doubles.nextDoc());
+ assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.longValue());
}
- NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", longs, cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean()));
+ NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Long.MAX_VALUE - i, longs.get(i));
+ assertEquals(i, longs.nextDoc());
+ assertEquals(Long.MAX_VALUE - i, longs.longValue());
}
- NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", ints, cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean()));
+ NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Integer.MAX_VALUE - i, ints.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(Integer.MAX_VALUE - i, ints.longValue());
}
- NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", floats, cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean()));
+ NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER);
for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.get(i));
+ assertEquals(i, floats.nextDoc());
+ assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.longValue());
}
Bits docsWithField = cache.getDocsWithField(reader, "theLong", null);
@@ -180,7 +184,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
IndexReader r = DirectoryReader.open(dir);
LeafReader reader = SlowCompositeReaderWrapper.wrap(r);
TestUtil.checkReader(reader);
- FieldCache.DEFAULT.getTerms(reader, "foobar", true);
+ FieldCache.DEFAULT.getTerms(reader, "foobar");
FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
r.close();
@@ -191,7 +195,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, true);
+ cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
// The double[] takes one slots, and docsWithField should also
// have been populated:
@@ -202,27 +206,20 @@ public class TestLegacyFieldCache extends LuceneTestCase {
assertEquals(2, cache.getCacheEntries().length);
assertTrue(bits instanceof Bits.MatchAllBits);
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true);
+ NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER);
assertEquals(4, cache.getCacheEntries().length);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.LEGACY_INT_PARSER);
- assertEquals(4, cache.getCacheEntries().length);
- for (int i = 0; i < docsWithField.length(); i++) {
+ for (int i = 0; i < reader.maxDoc(); i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(i, ints.longValue());
}
}
- NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean());
- docsWithField = cache.getDocsWithField(reader, "numInt", FieldCache.LEGACY_INT_PARSER);
- for (int i = 0; i < docsWithField.length(); i++) {
+ NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.LEGACY_INT_PARSER);
+ for (int i = 0; i < reader.maxDoc(); i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, numInts.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, numInts.nextDoc());
+ assertEquals(i, numInts.longValue());
}
}
}
@@ -265,14 +262,11 @@ public class TestLegacyFieldCache extends LuceneTestCase {
assertEquals(i%2 == 0, docsWithField.get(i));
}
} else {
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", null);
- for (int i = 0; i < docsWithField.length(); i++) {
+ NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER);
+ for (int i = 0; i < reader.maxDoc(); i++) {
if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
+ assertEquals(i, ints.nextDoc());
+ assertEquals(i, ints.longValue());
}
}
}
@@ -310,21 +304,22 @@ public class TestLegacyFieldCache extends LuceneTestCase {
// Binary type: can be retrieved via getTerms()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER);
});
// Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER);
});
// Numeric type: can be retrieved via getInts() and so on
- NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER, false);
- assertEquals(42, numeric.get(0));
+ NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER);
+ assertEquals(0, numeric.nextDoc());
+ assertEquals(42, numeric.longValue());
// SortedSet type: can be retrieved via getDocTermOrds()
expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER, false);
+ FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER);
});
ir.close();
@@ -345,17 +340,17 @@ public class TestLegacyFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true);
- assertEquals(0, ints.get(0));
+ NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER);
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true);
- assertEquals(0, longs.get(0));
+ NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER);
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true);
- assertEquals(0, floats.get(0));
+ NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER);
+ assertEquals(NO_MORE_DOCS, floats.nextDoc());
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true);
- assertEquals(0, doubles.get(0));
+ NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER);
+ assertEquals(NO_MORE_DOCS, doubles.nextDoc());
// check that we cached nothing
assertEquals(0, cache.getCacheEntries().length);
@@ -384,17 +379,17 @@ public class TestLegacyFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true);
- assertEquals(0, ints.get(0));
+ NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER);
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true);
- assertEquals(0, longs.get(0));
+ NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER);
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true);
- assertEquals(0, floats.get(0));
+ NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER);
+ assertEquals(NO_MORE_DOCS, floats.nextDoc());
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true);
- assertEquals(0, doubles.get(0));
+ NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER);
+ assertEquals(NO_MORE_DOCS, doubles.nextDoc());
// check that we cached nothing
assertEquals(0, cache.getCacheEntries().length);
@@ -412,6 +407,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
LegacyLongField field = new LegacyLongField("f", 0L, Store.YES);
doc.add(field);
final long[] values = new long[TestUtil.nextInt(random(), 1, 10)];
+ Set<Integer> missing = new HashSet<>();
for (int i = 0; i < values.length; ++i) {
final long v;
switch (random().nextInt(10)) {
@@ -432,6 +428,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
+ missing.add(i);
} else {
field.setLongValue(v);
iw.addDocument(doc);
@@ -439,10 +436,14 @@ public class TestLegacyFieldCache extends LuceneTestCase {
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
- final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_LONG_PARSER, false);
+ final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_LONG_PARSER);
for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], longs.get(i));
+ if (missing.contains(i) == false) {
+ assertEquals(i, longs.nextDoc());
+ assertEquals(values[i], longs.longValue());
+ }
}
+ assertEquals(NO_MORE_DOCS, longs.nextDoc());
reader.close();
iw.close();
dir.close();
@@ -458,6 +459,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
LegacyIntField field = new LegacyIntField("f", 0, Store.YES);
doc.add(field);
final int[] values = new int[TestUtil.nextInt(random(), 1, 10)];
+ Set<Integer> missing = new HashSet<>();
for (int i = 0; i < values.length; ++i) {
final int v;
switch (random().nextInt(10)) {
@@ -478,6 +480,7 @@ public class TestLegacyFieldCache extends LuceneTestCase {
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
+ missing.add(i);
} else {
field.setIntValue(v);
iw.addDocument(doc);
@@ -485,10 +488,14 @@ public class TestLegacyFieldCache extends LuceneTestCase {
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
- final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_INT_PARSER, false);
+ final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_INT_PARSER);
for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], ints.get(i));
+ if (missing.contains(i) == false) {
+ assertEquals(i, ints.nextDoc());
+ assertEquals(values[i], ints.longValue());
+ }
}
+ assertEquals(NO_MORE_DOCS, ints.nextDoc());
reader.close();
iw.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
index c0188e4..e9e94e2 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
@@ -75,11 +75,11 @@ public class TestUninvertingReader extends LuceneTestCase {
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
@@ -117,11 +117,11 @@ public class TestUninvertingReader extends LuceneTestCase {
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
@@ -158,11 +158,11 @@ public class TestUninvertingReader extends LuceneTestCase {
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
@@ -199,11 +199,11 @@ public class TestUninvertingReader extends LuceneTestCase {
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
index 6e6b30e..2c0b634 100644
--- a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
+++ b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
@@ -360,15 +360,18 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
NumericDocValues fooNorms = reader.getNormValues("foo_t");
NumericDocValues textNorms = reader.getNormValues("text");
+ assertEquals(docid, titleNorms.advance(docid));
assertEquals(expectedNorm(sim, 2, TITLE_BOOST * DOC_BOOST),
- titleNorms.get(docid));
+ titleNorms.longValue());
+ assertEquals(docid, fooNorms.advance(docid));
assertEquals(expectedNorm(sim, 8-3, FOO_BOOST * DOC_BOOST),
- fooNorms.get(docid));
+ fooNorms.longValue());
+ assertEquals(docid, textNorms.advance(docid));
assertEquals(expectedNorm(sim, 2 + 8-3,
TITLE_BOOST * FOO_BOOST * DOC_BOOST),
- textNorms.get(docid));
+ textNorms.longValue());
} finally {
req.close();
[05/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
index 85ac12f..b9bf745 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
@@ -25,8 +25,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Map.Entry;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
@@ -49,22 +49,25 @@ import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/**
* Abstract class to do basic tests for a docvalues format.
@@ -83,17 +86,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
}
- if (codecSupportsSortedSet()) {
- final int numValues = random().nextInt(5);
- for (int i = 0; i < numValues; ++i) {
- doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
- }
+ int numValues = random().nextInt(5);
+ for (int i = 0; i < numValues; ++i) {
+ doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
}
- if (codecSupportsSortedNumeric()) {
- final int numValues = random().nextInt(5);
- for (int i = 0; i < numValues; ++i) {
- doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
- }
+ numValues = random().nextInt(5);
+ for (int i = 0; i < numValues; ++i) {
+ doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
}
}
@@ -122,7 +121,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
- assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ int docID = hits.scoreDocs[i].doc;
+ assertEquals(docID, dv.advance(docID));
+ assertEquals(5, dv.longValue());
}
ireader.close();
@@ -150,11 +151,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
+
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
- assertEquals(Float.floatToRawIntBits(5.7f), dv.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv.advance(docID));
+ assertEquals(Float.floatToRawIntBits(5.7f), dv.longValue());
}
ireader.close();
@@ -183,13 +187,16 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
- assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv.advance(docID));
+ assertEquals(5, dv.longValue());
dv = ireader.leaves().get(0).reader().getNumericDocValues("dv2");
- assertEquals(17, dv.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv.advance(docID));
+ assertEquals(17, dv.longValue());
}
ireader.close();
@@ -218,14 +225,17 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int hitDocID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(hitDocID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv1");
- BytesRef scratch = dv.get(hits.scoreDocs[i].doc);
+ assertEquals(hitDocID, dv.advance(hitDocID));
+ BytesRef scratch = dv.binaryValue();
assertEquals(new BytesRef(longTerm), scratch);
dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
- scratch = dv.get(hits.scoreDocs[i].doc);
+ assertEquals(hitDocID, dv.advance(hitDocID));
+ scratch = dv.binaryValue();
assertEquals(new BytesRef(text), scratch);
}
@@ -255,14 +265,16 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
- assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv.advance(docID));
+ assertEquals(5, dv.longValue());
BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
- BytesRef scratch = dv2.get(hits.scoreDocs[i].doc);
- assertEquals(new BytesRef("hello world"), scratch);
+ assertEquals(docID, dv2.advance(docID));
+ assertEquals(new BytesRef("hello world"), dv2.binaryValue());
}
ireader.close();
@@ -292,18 +304,21 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv1");
- int ord = dv.getOrd(0);
+ assertEquals(docID, dv.advance(docID));
+ int ord = dv.ordValue();
BytesRef scratch = dv.lookupOrd(ord);
assertEquals(new BytesRef("hello hello"), scratch);
NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv2");
- assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv2.advance(docID));
+ assertEquals(5, dv2.longValue());
BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv3");
- scratch = dv3.get(hits.scoreDocs[i].doc);
- assertEquals(new BytesRef("hello world"), scratch);
+ assertEquals(docID, dv3.advance(docID));
+ assertEquals(new BytesRef("hello world"), dv3.binaryValue());
}
ireader.close();
@@ -334,18 +349,21 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv2");
- int ord = dv.getOrd(0);
+ assertEquals(docID, dv.advance(docID));
+ int ord = dv.ordValue();
scratch = dv.lookupOrd(ord);
assertEquals(new BytesRef("hello hello"), scratch);
NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv3");
- assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv2.advance(docID));
+ assertEquals(5, dv2.longValue());
BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv1");
- scratch = dv3.get(hits.scoreDocs[i].doc);
- assertEquals(new BytesRef("hello world"), scratch);
+ assertEquals(docID, dv3.advance(docID));
+ assertEquals(new BytesRef("hello world"), dv3.binaryValue());
}
ireader.close();
@@ -372,8 +390,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
- assertEquals(1, dv.get(0));
- assertEquals(2, dv.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(1, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(2, dv.longValue());
ireader.close();
directory.close();
@@ -410,7 +430,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
expected = 99;
}
- assertEquals(expected, dv.get(i));
+ assertEquals(i, dv.nextDoc());
+ assertEquals(expected, dv.longValue());
}
ireader.close();
@@ -437,8 +458,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
- assertEquals(Long.MIN_VALUE, dv.get(0));
- assertEquals(Long.MAX_VALUE, dv.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(Long.MIN_VALUE, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(Long.MAX_VALUE, dv.longValue());
ireader.close();
directory.close();
@@ -464,8 +487,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
- assertEquals(-8841491950446638677L, dv.get(0));
- assertEquals(9062230939892376225L, dv.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(-8841491950446638677L, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(9062230939892376225L, dv.longValue());
ireader.close();
directory.close();
@@ -495,12 +520,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int hitDocID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(hitDocID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- BytesRef scratch = dv.get(hits.scoreDocs[i].doc);
- assertEquals(new BytesRef("hello world"), scratch);
+ assertEquals(hitDocID, dv.advance(hitDocID));
+ assertEquals(new BytesRef("hello world"), dv.binaryValue());
}
ireader.close();
@@ -539,8 +565,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
expected = "hello 2";
}
- scratch = dv.get(i);
- assertEquals(expected, scratch.utf8ToString());
+ assertEquals(i, dv.nextDoc());
+ assertEquals(expected, dv.binaryValue().utf8ToString());
}
ireader.close();
@@ -569,7 +595,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
BinaryDocValues dv = getOnlyLeafReader(ireader).getBinaryDocValues("field");
- assertEquals(new BytesRef(), dv.get(0));
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -600,11 +626,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int docID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(docID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
- scratch = dv.lookupOrd(dv.getOrd(hits.scoreDocs[i].doc));
+ assertEquals(docID, dv.advance(docID));
+ scratch = dv.lookupOrd(dv.ordValue());
assertEquals(new BytesRef("hello world"), scratch);
}
@@ -633,9 +661,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
- scratch = dv.lookupOrd(dv.getOrd(0));
+ assertEquals(0, dv.nextDoc());
+ scratch = dv.lookupOrd(dv.ordValue());
assertEquals("hello world 1", scratch.utf8ToString());
- scratch = dv.lookupOrd(dv.getOrd(1));
+ assertEquals(1, dv.nextDoc());
+ scratch = dv.lookupOrd(dv.ordValue());
assertEquals("hello world 2", scratch.utf8ToString());
ireader.close();
@@ -666,13 +696,16 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
assertEquals(2, dv.getValueCount());
- assertEquals(0, dv.getOrd(0));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(0, dv.ordValue());
BytesRef scratch = dv.lookupOrd(0);
assertEquals("hello world 1", scratch.utf8ToString());
- assertEquals(1, dv.getOrd(1));
+ assertEquals(1, dv.nextDoc());
+ assertEquals(1, dv.ordValue());
scratch = dv.lookupOrd(1);
assertEquals("hello world 2", scratch.utf8ToString());
- assertEquals(0, dv.getOrd(2));
+ assertEquals(2, dv.nextDoc());
+ assertEquals(0, dv.ordValue());
ireader.close();
directory.close();
@@ -702,7 +735,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
assertEquals(2, dv.getValueCount()); // 2 ords
- BytesRef scratch = dv.lookupOrd(0);
+ assertEquals(0, dv.nextDoc());
+ BytesRef scratch = dv.binaryValue();
assertEquals(new BytesRef("hello world 1"), scratch);
scratch = dv.lookupOrd(1);
assertEquals(new BytesRef("hello world 2"), scratch);
@@ -714,7 +748,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
expected = "hello world 2";
}
- scratch = dv.lookupOrd(dv.getOrd(i));
+ if (dv.docID() < i) {
+ assertEquals(i, dv.nextDoc());
+ }
+ scratch = dv.lookupOrd(dv.ordValue());
assertEquals(expected, scratch.utf8ToString());
}
@@ -744,15 +781,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field");
- if (codecSupportsDocsWithField()) {
- assertEquals(-1, dv.getOrd(0));
- assertEquals(0, dv.getValueCount());
- } else {
- assertEquals(0, dv.getOrd(0));
- assertEquals(1, dv.getValueCount());
- BytesRef ref = dv.lookupOrd(0);
- assertEquals(new BytesRef(), ref);
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -774,8 +803,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- BytesRef scratch = dv.get(0);
- assertEquals(new BytesRef("hello\nworld\r1"), scratch);
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef("hello\nworld\r1"), dv.binaryValue());
ireader.close();
directory.close();
@@ -799,13 +828,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
- BytesRef scratch = dv.lookupOrd(dv.getOrd(0));
+ assertEquals(0, dv.nextDoc());
+ BytesRef scratch = dv.lookupOrd(dv.ordValue());
assertEquals(new BytesRef("hello world 2"), scratch);
- if (codecSupportsDocsWithField()) {
- assertEquals(-1, dv.getOrd(1));
- }
- scratch = dv.get(1);
- assertEquals(new BytesRef(""), scratch);
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
}
@@ -903,9 +929,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
- assertEquals(0, dv.getOrd(0));
- assertEquals(0, dv.getOrd(1));
- BytesRef scratch = dv.lookupOrd(dv.getOrd(0));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(0, dv.ordValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(0, dv.ordValue());
+ BytesRef scratch = dv.lookupOrd(0);
assertEquals("", scratch.utf8ToString());
ireader.close();
@@ -932,10 +960,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- BytesRef scratch = dv.get(0);
- assertEquals("", scratch.utf8ToString());
- scratch = dv.get(1);
- assertEquals("", scratch.utf8ToString());
+ assertEquals(0, dv.nextDoc());
+ assertEquals("", dv.binaryValue().utf8ToString());
+ assertEquals(1, dv.nextDoc());
+ assertEquals("", dv.binaryValue().utf8ToString());
ireader.close();
directory.close();
@@ -960,8 +988,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- BytesRef scratch = dv.get(0);
- assertEquals(new BytesRef(bytes), scratch);
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef(bytes), dv.binaryValue());
ireader.close();
directory.close();
@@ -985,9 +1013,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
- BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
- BytesRef scratch = dv.get(0);
- assertEquals(new BytesRef(bytes), scratch);
+ BinaryDocValues dv = DocValues.getBinary(ireader.leaves().get(0).reader(), "dv");
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef(bytes), dv.binaryValue());
ireader.close();
directory.close();
}
@@ -1008,10 +1036,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- byte mybytes[] = new byte[20];
- BytesRef scratch = dv.get(0);
- assertEquals("boo!", scratch.utf8ToString());
- assertFalse(scratch.bytes == mybytes);
+ assertEquals(0, dv.nextDoc());
+ assertEquals("boo!", dv.binaryValue().utf8ToString());
ireader.close();
directory.close();
@@ -1032,11 +1058,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
- BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BinaryDocValues dv = DocValues.getBinary(ireader.leaves().get(0).reader(), "dv");
byte mybytes[] = new byte[20];
- BytesRef scratch = dv.get(0);
- assertEquals("boo!", scratch.utf8ToString());
- assertFalse(scratch.bytes == mybytes);
+ assertEquals(0, dv.nextDoc());
+ assertEquals("boo!", dv.binaryValue().utf8ToString());
+ assertFalse(dv.binaryValue().bytes == mybytes);
ireader.close();
directory.close();
@@ -1080,7 +1106,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
NumericDocValues docValues = getOnlyLeafReader(reader).getNumericDocValues("docId");
for (int i = 0; i < scoreDocs.length; i++) {
assertEquals(i, scoreDocs[i].doc);
- assertEquals(i, docValues.get(scoreDocs[i].doc));
+ assertEquals(i, docValues.advance(i));
+ assertEquals(i, docValues.longValue());
}
reader.close();
dir.close();
@@ -1089,11 +1116,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
public void testRandomSortedBytes() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
- if (!codecSupportsDocsWithField()) {
- // if the codec doesnt support missing, we expect missing to be mapped to byte[]
- // by the impersonator, but we have to give it a chance to merge them to this
- cfg.setMergePolicy(newLogMergePolicy());
- }
RandomIndexWriter w = new RandomIndexWriter(random(), dir, cfg);
int numDocs = atLeast(100);
BytesRefHash hash = new BytesRefHash();
@@ -1118,18 +1140,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
doc.add(newTextField("id", "noValue", Field.Store.YES));
w.addDocument(doc);
}
- if (!codecSupportsDocsWithField()) {
- BytesRef bytesRef = new BytesRef();
- hash.add(bytesRef); // add empty value for the gaps
- }
if (rarely()) {
w.commit();
}
- if (!codecSupportsDocsWithField()) {
- // if the codec doesnt support missing, we expect missing to be mapped to byte[]
- // by the impersonator, but we have to give it a chance to merge them to this
- w.forceMerge(1);
- }
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
String id = "" + i + numDocs;
@@ -1161,7 +1174,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
PostingsEnum termPostingsEnum = TestUtil.docs(random(), reader, "id", new BytesRef(entry.getKey()), null, 0);
int docId = termPostingsEnum.nextDoc();
expected = new BytesRef(entry.getValue());
- final BytesRef actual = docValues.get(docId);
+ docValues = MultiDocValues.getSortedValues(reader, "field");
+ assertEquals(docId, docValues.advance(docId));
+ final BytesRef actual = docValues.binaryValue();
assertEquals(expected, actual);
}
@@ -1231,7 +1246,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
NumericDocValues docValues = r.getNumericDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
long storedValue = Long.parseLong(r.document(i).get("stored"));
- assertEquals(storedValue, docValues.get(i));
+ assertEquals(i, docValues.nextDoc());
+ assertEquals(storedValue, docValues.longValue());
}
}
ir.close();
@@ -1288,13 +1304,19 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
LeafReader r = context.reader();
SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
for (int i = 0; i < r.maxDoc(); i++) {
+ if (i > docValues.docID()) {
+ docValues.nextDoc();
+ }
String expected[] = r.document(i).getValues("stored");
- docValues.setDocument(i);
- String actual[] = new String[docValues.count()];
- for (int j = 0; j < actual.length; j++) {
- actual[j] = Long.toString(docValues.valueAt(j));
+ if (i < docValues.docID()) {
+ assertEquals(0, expected.length);
+ } else {
+ String actual[] = new String[docValues.docValueCount()];
+ for (int j = 0; j < actual.length; j++) {
+ actual[j] = Long.toString(docValues.nextValue());
+ }
+ assertArrayEquals(expected, actual);
}
- assertArrayEquals(expected, actual);
}
}
ir.close();
@@ -1382,8 +1404,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BinaryDocValues docValues = r.getBinaryDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
- BytesRef scratch = docValues.get(i);
- assertEquals(binaryValue, scratch);
+ assertEquals(i, docValues.nextDoc());
+ assertEquals(binaryValue, docValues.binaryValue());
}
}
ir.close();
@@ -1396,8 +1418,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BinaryDocValues docValues = r.getBinaryDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
- BytesRef scratch = docValues.get(i);
- assertEquals(binaryValue, scratch);
+ assertEquals(i, docValues.nextDoc());
+ assertEquals(binaryValue, docValues.binaryValue());
}
}
ir.close();
@@ -1462,11 +1484,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
DirectoryReader ir = writer.getReader();
for (LeafReaderContext context : ir.leaves()) {
LeafReader r = context.reader();
- BinaryDocValues docValues = r.getSortedDocValues("dv");
+ BinaryDocValues docValues = DocValues.getBinary(r, "dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
- BytesRef scratch = docValues.get(i);
- assertEquals(binaryValue, scratch);
+ assertEquals(i, docValues.nextDoc());
+ assertEquals(binaryValue, docValues.binaryValue());
}
}
ir.close();
@@ -1476,11 +1498,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
ir = writer.getReader();
for (LeafReaderContext context : ir.leaves()) {
LeafReader r = context.reader();
- BinaryDocValues docValues = r.getSortedDocValues("dv");
+ BinaryDocValues docValues = DocValues.getBinary(r, "dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
- BytesRef scratch = docValues.get(i);
- assertEquals(binaryValue, scratch);
+ assertEquals(i, docValues.nextDoc());
+ assertEquals(binaryValue, docValues.binaryValue());
}
}
ir.close();
@@ -1504,7 +1526,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetOneValue() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
@@ -1516,8 +1537,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
-
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1529,7 +1549,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoFields() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
@@ -1542,8 +1561,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
+ assertEquals(0, dv.nextDoc());
- dv.setDocument(0);
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1552,7 +1571,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field2");
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1564,7 +1583,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoDocumentsMerged() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1587,14 +1605,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(2, dv.getValueCount());
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
- dv.setDocument(1);
+ assertEquals(1, dv.nextDoc());
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1606,7 +1624,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoValues() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
@@ -1619,8 +1636,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
+ assertEquals(0, dv.nextDoc());
- dv.setDocument(0);
assertEquals(0, dv.nextOrd());
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1636,7 +1653,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoValuesUnordered() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
@@ -1649,8 +1665,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
+ assertEquals(0, dv.nextDoc());
- dv.setDocument(0);
assertEquals(0, dv.nextOrd());
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1666,7 +1682,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetThreeValuesTwoDocs() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1691,12 +1706,12 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(3, dv.getValueCount());
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(1, dv.nextOrd());
assertEquals(2, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
- dv.setDocument(1);
+ assertEquals(1, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1715,7 +1730,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoDocumentsLastMissing() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1735,7 +1749,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(1, dv.getValueCount());
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1747,7 +1761,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoDocumentsLastMissingMerge() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1769,7 +1782,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(1, dv.getValueCount());
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1781,7 +1794,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoDocumentsFirstMissing() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1802,7 +1814,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(1, dv.getValueCount());
- dv.setDocument(1);
+ assertEquals(1, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1814,7 +1826,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTwoDocumentsFirstMissingMerge() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1836,7 +1847,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
assertEquals(1, dv.getValueCount());
- dv.setDocument(1);
+ assertEquals(1, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
@@ -1848,7 +1859,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetMergeAwayAllValues() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1877,7 +1887,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetTermsEnum() throws IOException {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -1998,16 +2007,21 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
for (int i = 0; i < r.maxDoc(); i++) {
String stringValues[] = r.document(i).getValues("stored");
if (docValues != null) {
- docValues.setDocument(i);
+ if (docValues.docID() < i) {
+ docValues.nextDoc();
+ }
}
- for (int j = 0; j < stringValues.length; j++) {
- assert docValues != null;
- long ord = docValues.nextOrd();
- assert ord != NO_MORE_ORDS;
- BytesRef scratch = docValues.lookupOrd(ord);
- assertEquals(stringValues[j], scratch.utf8ToString());
+ if (docValues != null && stringValues.length > 0) {
+ assertEquals(i, docValues.docID());
+ for (int j = 0; j < stringValues.length; j++) {
+ assert docValues != null;
+ long ord = docValues.nextOrd();
+ assert ord != NO_MORE_ORDS;
+ BytesRef scratch = docValues.lookupOrd(ord);
+ assertEquals(stringValues[j], scratch.utf8ToString());
+ }
+ assertEquals(NO_MORE_ORDS, docValues.nextOrd());
}
- assert docValues == null || docValues.nextOrd() == NO_MORE_ORDS;
}
}
ir.close();
@@ -2020,17 +2034,20 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedSetDocValues docValues = r.getSortedSetDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
String stringValues[] = r.document(i).getValues("stored");
- if (docValues != null) {
- docValues.setDocument(i);
+ if (docValues.docID() < i) {
+ docValues.nextDoc();
}
- for (int j = 0; j < stringValues.length; j++) {
- assert docValues != null;
- long ord = docValues.nextOrd();
- assert ord != NO_MORE_ORDS;
- BytesRef scratch = docValues.lookupOrd(ord);
- assertEquals(stringValues[j], scratch.utf8ToString());
+ if (docValues != null && stringValues.length > 0) {
+ assertEquals(i, docValues.docID());
+ for (int j = 0; j < stringValues.length; j++) {
+ assert docValues != null;
+ long ord = docValues.nextOrd();
+ assert ord != NO_MORE_ORDS;
+ BytesRef scratch = docValues.lookupOrd(ord);
+ assertEquals(stringValues[j], scratch.utf8ToString());
+ }
+ assertEquals(NO_MORE_ORDS, docValues.nextOrd());
}
- assert docValues == null || docValues.nextOrd() == NO_MORE_ORDS;
}
}
ir.close();
@@ -2039,7 +2056,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetFixedLengthVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
int fixedLength = TestUtil.nextInt(random(), 1, 10);
@@ -2048,7 +2064,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedNumericsSingleValuedVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedNumericsVsStoredFields(
@@ -2069,7 +2084,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedNumericsSingleValuedMissingVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedNumericsVsStoredFields(
@@ -2090,7 +2104,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedNumericsMultipleValuesVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedNumericsVsStoredFields(
@@ -2111,7 +2124,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedNumericsFewUniqueSetsVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
final long[] values = new long[TestUtil.nextInt(random(), 2, 6)];
for (int i = 0; i < values.length; ++i) {
values[i] = random().nextLong();
@@ -2136,7 +2148,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetVariableLengthVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(300), 1, 10, 16, 100);
@@ -2144,7 +2155,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetFixedLengthSingleValuedVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
int fixedLength = TestUtil.nextInt(random(), 1, 10);
@@ -2153,7 +2163,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetVariableLengthSingleValuedVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(300), 1, 10, 1, 100);
@@ -2161,7 +2170,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetFixedLengthFewUniqueSetsVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(300), 10, 10, 6, 6);
@@ -2169,7 +2177,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetVariableLengthFewUniqueSetsVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(300), 1, 10, 6, 6);
@@ -2177,7 +2184,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetVariableLengthManyValuesPerDocVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(20), 1, 10, 500, 1000);
@@ -2185,7 +2191,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testSortedSetFixedLengthManyValuesPerDocVsStoredFields() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
doTestSortedSetVsStoredFields(atLeast(20), 10, 10, 500, 1000);
@@ -2227,7 +2232,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testTwoNumbersOneMissing() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2246,17 +2250,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
NumericDocValues dv = ar.getNumericDocValues("dv1");
- assertEquals(0, dv.get(0));
- assertEquals(0, dv.get(1));
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(0, dv.longValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ir.close();
directory.close();
}
public void testTwoNumbersOneMissingWithMerging() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2276,17 +2277,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
NumericDocValues dv = ar.getNumericDocValues("dv1");
- assertEquals(0, dv.get(0));
- assertEquals(0, dv.get(1));
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(0, dv.longValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ir.close();
directory.close();
}
public void testThreeNumbersOneMissingWithMerging() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2310,19 +2308,15 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
NumericDocValues dv = ar.getNumericDocValues("dv1");
- assertEquals(0, dv.get(0));
- assertEquals(0, dv.get(1));
- assertEquals(5, dv.get(2));
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
- assertTrue(docsWithField.get(2));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(0, dv.longValue());
+ assertEquals(2, dv.nextDoc());
+ assertEquals(5, dv.longValue());
ir.close();
directory.close();
}
public void testTwoBytesOneMissing() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2341,19 +2335,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
- BytesRef ref = dv.get(0);
- assertEquals(new BytesRef(), ref);
- ref = dv.get(1);
- assertEquals(new BytesRef(), ref);
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef(), dv.binaryValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ir.close();
directory.close();
}
public void testTwoBytesOneMissingWithMerging() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2373,19 +2362,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
- BytesRef ref = dv.get(0);
- assertEquals(new BytesRef(), ref);
- ref = dv.get(1);
- assertEquals(new BytesRef(), ref);
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef(), dv.binaryValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ir.close();
directory.close();
}
public void testThreeBytesOneMissingWithMerging() throws IOException {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setMergePolicy(newLogMergePolicy());
@@ -2409,16 +2393,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
LeafReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
- BytesRef ref = dv.get(0);
- assertEquals(new BytesRef(), ref);
- ref = dv.get(1);
- assertEquals(new BytesRef(), ref);
- ref = dv.get(2);
- assertEquals(new BytesRef("boo"), ref);
- Bits docsWithField = ar.getDocsWithField("dv1");
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
- assertTrue(docsWithField.get(2));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(new BytesRef(), dv.binaryValue());
+ assertEquals(2, dv.nextDoc());
+ assertEquals(new BytesRef("boo"), dv.binaryValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ir.close();
directory.close();
}
@@ -2488,12 +2467,15 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
NumericDocValues numerics = r.getNumericDocValues("dvNum");
for (int j = 0; j < r.maxDoc(); j++) {
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
- BytesRef scratch = binaries.get(j);
+ assertEquals(j, binaries.nextDoc());
+ BytesRef scratch = binaries.binaryValue();
assertEquals(binaryValue, scratch);
- scratch = sorted.get(j);
+ assertEquals(j, sorted.nextDoc());
+ scratch = sorted.binaryValue();
assertEquals(binaryValue, scratch);
String expected = r.document(j).get("storedNum");
- assertEquals(Long.parseLong(expected), numerics.get(j));
+ assertEquals(j, numerics.nextDoc());
+ assertEquals(Long.parseLong(expected), numerics.longValue());
}
}
TestUtil.checkReader(ir);
@@ -2515,9 +2497,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
/** Tests dv against stored fields with threads (all types + missing) */
@Slow
public void testThreads2() throws Exception {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
@@ -2599,46 +2578,35 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
for (LeafReaderContext context : ir.leaves()) {
LeafReader r = context.reader();
BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
- Bits binaryBits = r.getDocsWithField("dvBin");
SortedDocValues sorted = r.getSortedDocValues("dvSorted");
- Bits sortedBits = r.getDocsWithField("dvSorted");
NumericDocValues numerics = r.getNumericDocValues("dvNum");
- Bits numericBits = r.getDocsWithField("dvNum");
SortedSetDocValues sortedSet = r.getSortedSetDocValues("dvSortedSet");
- Bits sortedSetBits = r.getDocsWithField("dvSortedSet");
SortedNumericDocValues sortedNumeric = r.getSortedNumericDocValues("dvSortedNumeric");
- Bits sortedNumericBits = r.getDocsWithField("dvSortedNumeric");
for (int j = 0; j < r.maxDoc(); j++) {
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
if (binaryValue != null) {
if (binaries != null) {
- BytesRef scratch = binaries.get(j);
+ assertEquals(j, binaries.nextDoc());
+ BytesRef scratch = binaries.binaryValue();
assertEquals(binaryValue, scratch);
- scratch = sorted.get(j);
+ assertEquals(j, sorted.nextDoc());
+ scratch = sorted.binaryValue();
assertEquals(binaryValue, scratch);
- assertTrue(binaryBits.get(j));
- assertTrue(sortedBits.get(j));
}
- } else if (binaries != null) {
- assertFalse(binaryBits.get(j));
- assertFalse(sortedBits.get(j));
- assertEquals(-1, sorted.getOrd(j));
}
String number = r.document(j).get("storedNum");
if (number != null) {
if (numerics != null) {
- assertEquals(Long.parseLong(number), numerics.get(j));
+ assertEquals(j, numerics.advance(j));
+ assertEquals(Long.parseLong(number), numerics.longValue());
}
- } else if (numerics != null) {
- assertFalse(numericBits.get(j));
- assertEquals(0, numerics.get(j));
}
String values[] = r.document(j).getValues("storedSortedSet");
if (values.length > 0) {
assertNotNull(sortedSet);
- sortedSet.setDocument(j);
+ assertEquals(j, sortedSet.nextDoc());
for (int k = 0; k < values.length; k++) {
long ord = sortedSet.nextOrd();
assertTrue(ord != SortedSetDocValues.NO_MORE_ORDS);
@@ -2646,27 +2614,17 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(values[k], value.utf8ToString());
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
- assertTrue(sortedSetBits.get(j));
- } else if (sortedSet != null) {
- sortedSet.setDocument(j);
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
- assertFalse(sortedSetBits.get(j));
}
String numValues[] = r.document(j).getValues("storedSortedNumeric");
if (numValues.length > 0) {
assertNotNull(sortedNumeric);
- sortedNumeric.setDocument(j);
- assertEquals(numValues.length, sortedNumeric.count());
+ assertEquals(j, sortedNumeric.nextDoc());
+ assertEquals(numValues.length, sortedNumeric.docValueCount());
for (int k = 0; k < numValues.length; k++) {
- long v = sortedNumeric.valueAt(k);
+ long v = sortedNumeric.nextValue();
assertEquals(numValues[k], Long.toString(v));
}
- assertTrue(sortedNumericBits.get(j));
- } else if (sortedNumeric != null) {
- sortedNumeric.setDocument(j);
- assertEquals(0, sortedNumeric.count());
- assertFalse(sortedNumericBits.get(j));
}
}
}
@@ -2688,9 +2646,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
@Slow
public void testThreads3() throws Exception {
- assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField());
- assumeTrue("Codec does not support SORTED_SET", codecSupportsSortedSet());
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory dir = newFSDirectory(createTempDir());
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
@@ -2785,7 +2740,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BinaryDocValues values = MultiDocValues.getBinaryValues(r, "field");
for(int j=0;j<5;j++) {
- BytesRef result = values.get(0);
+ assertEquals(j, values.nextDoc());
+ BytesRef result = values.binaryValue();
assertTrue(result.length == 0 || result.length == 1<<i);
}
r.close();
@@ -2794,7 +2750,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
}
public void testOneSortedNumber() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
Document doc = new Document();
@@ -2806,16 +2761,15 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(1, dv.count());
- assertEquals(5, dv.valueAt(0));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(1, dv.docValueCount());
+ assertEquals(5, dv.nextValue());
reader.close();
directory.close();
}
public void testOneSortedNumberOneMissing() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(null));
Document doc = new Document();
@@ -2828,17 +2782,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(1, dv.count());
- assertEquals(5, dv.valueAt(0));
- dv.setDocument(1);
- assertEquals(0, dv.count());
+ assertEquals(0, dv.nextDoc());
+ assertEquals(1, dv.docValueCount());
+ assertEquals(5, dv.nextValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
- Bits docsWithField = reader.leaves().get(0).reader().getDocsWithField("dv");
- assertEquals(2, docsWithField.length());
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
-
reader.close();
directory.close();
}
@@ -2865,14 +2813,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
NumericDocValues dv = getOnlyLeafReader(ireader).getNumericDocValues("field");
- assertEquals(0, dv.get(0));
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
}
public void testTwoSortedNumber() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
Document doc = new Document();
@@ -2885,17 +2832,16 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(2, dv.count());
- assertEquals(-5, dv.valueAt(0));
- assertEquals(11, dv.valueAt(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(2, dv.docValueCount());
+ assertEquals(-5, dv.nextValue());
+ assertEquals(11, dv.nextValue());
reader.close();
directory.close();
}
public void testTwoSortedNumberSameValue() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
Document doc = new Document();
@@ -2908,17 +2854,16 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(2, dv.count());
- assertEquals(11, dv.valueAt(0));
- assertEquals(11, dv.valueAt(1));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(2, dv.docValueCount());
+ assertEquals(11, dv.nextValue());
+ assertEquals(11, dv.nextValue());
reader.close();
directory.close();
}
public void testTwoSortedNumberOneMissing() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(null));
Document doc = new Document();
@@ -2932,24 +2877,17 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(2, dv.count());
- assertEquals(-5, dv.valueAt(0));
- assertEquals(11, dv.valueAt(1));
- dv.setDocument(1);
- assertEquals(0, dv.count());
-
- Bits docsWithField = reader.leaves().get(0).reader().getDocsWithField("dv");
- assertEquals(2, docsWithField.length());
- assertTrue(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
-
+ assertEquals(0, dv.nextDoc());
+ assertEquals(2, dv.docValueCount());
+ assertEquals(-5, dv.nextValue());
+ assertEquals(11, dv.nextValue());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
+
reader.close();
directory.close();
}
public void testSortedNumberMerge() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setMergePolicy(newLogMergePolicy());
@@ -2968,19 +2906,18 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader reader = DirectoryReader.open(directory);
assert reader.leaves().size() == 1;
SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
- dv.setDocument(0);
- assertEquals(1, dv.count());
- assertEquals(11, dv.valueAt(0));
- dv.setDocument(1);
- assertEquals(1, dv.count());
- assertEquals(-5, dv.valueAt(0));
+ assertEquals(0, dv.nextDoc());
+ assertEquals(1, dv.docValueCount());
+ assertEquals(11, dv.nextValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(1, dv.docValueCount());
+ assertEquals(-5, dv.nextValue());
reader.close();
directory.close();
}
public void testSortedNumberMergeAwayAllValues() throws IOException {
- assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
@@ -3002,8 +2939,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedNumericDocValues dv = getOnlyLeafReader(ireader).getSortedNumericDocValues("field");
- dv.setDocument(0);
- assertEquals(0, dv.count());
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -3169,9 +3105,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field");
- for (int i = 0; i < numEmptyDocs; ++i) {
- assertEquals(-1, dv.getOrd(i));
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -3201,10 +3135,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
- for (int i = 0; i < numEmptyDocs; ++i) {
- dv.setDocument(i);
- assertEquals(-1L, dv.nextOrd());
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -3234,11 +3165,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
NumericDocValues dv = getOnlyLeafReader(ireader).getNumericDocValues("field");
- Bits docsWithField = getOnlyLeafReader(ireader).getDocsWithField("field");
- for (int i = 0; i < numEmptyDocs; ++i) {
- assertEquals(0, dv.get(i));
- assertFalse(docsWithField.get(i));
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -3268,10 +3195,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
SortedNumericDocValues dv = getOnlyLeafReader(ireader).getSortedNumericDocValues("field");
- for (int i = 0; i < numEmptyDocs; ++i) {
- dv.setDocument(i);
- assertEquals(0, dv.count());
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
@@ -3301,34 +3225,126 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
iwriter.close();
BinaryDocValues dv = getOnlyLeafReader(ireader).getBinaryDocValues("field");
- Bits docsWithField = getOnlyLeafReader(ireader).getDocsWithField("field");
- for (int i = 0; i < numEmptyDocs; ++i) {
- assertEquals(new BytesRef(), dv.get(i));
- assertFalse(docsWithField.get(i));
- }
+ assertEquals(NO_MORE_DOCS, dv.nextDoc());
ireader.close();
directory.close();
}
- protected boolean codecAcceptsHugeBinaryValues(String field) {
- return true;
+ public void testRandomAdvanceNumeric() throws IOException {
+ final long longRange;
+ if (random().nextBoolean()) {
+ longRange = TestUtil.nextInt(random(), 1, 1024);
+ } else {
+ longRange = TestUtil.nextLong(random(), 1, Long.MAX_VALUE);
+ }
+ doTestRandomAdvance(new FieldCreator() {
+ @Override
+ public Field next() {
+ return new NumericDocValuesField("field", TestUtil.nextLong(random(), 0, longRange));
+ }
+
+ @Override
+ public DocIdSetIterator iterator(IndexReader r) throws IOException {
+ return MultiDocValues.getNumericValues(r, "field");
+ }
+ });
}
-
- /** Returns true if the codec "supports" docsWithField
- * (other codecs return MatchAllBits, because you couldnt write missing values before) */
- protected boolean codecSupportsDocsWithField() {
- return true;
+
+ public void testRandomAdvanceBinary() throws IOException {
+ doTestRandomAdvance(new FieldCreator() {
+ @Override
+ public Field next() {
+ byte[] bytes = new byte[random().nextInt(10)];
+ random().nextBytes(bytes);
+ return new BinaryDocValuesField("field", new BytesRef(bytes));
+ }
+
+ @Override
+ public DocIdSetIterator iterator(IndexReader r) throws IOException {
+ return MultiDocValues.getBinaryValues(r, "field");
+ }
+ });
}
-
- /** Returns true if the default codec supports SORTED_SET docvalues */
- protected boolean codecSupportsSortedSet() {
- return true;
+
+ private interface FieldCreator {
+ public Field next();
+ public DocIdSetIterator iterator(IndexReader r) throws IOException;
}
-
- /** Returns true if the default codec supports SORTED_NUMERIC docvalues */
- protected boolean codecSupportsSortedNumeric() {
+
+ private void doTestRandomAdvance(FieldCreator fieldCreator) throws IOException {
+
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(analyzer);
+ conf.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter w = new RandomIndexWriter(random(), directory, conf);
+ int numChunks = atLeast(10);
+ int id = 0;
+ Set<Integer> missingSet = new HashSet<>();
+ for(int i=0;i<numChunks;i++) {
+ // change sparseness for each chunk
+ double sparseChance = random().nextDouble();
+ int docCount = atLeast(1000);
+ for(int j=0;j<docCount;j++) {
+ Document doc = new Document();
+ doc.add(new StoredField("id", id));
+ if (random().nextDouble() > sparseChance) {
+ doc.add(fieldCreator.next());
+ } else {
+ missingSet.add(id);
+ }
+ id++;
+ w.addDocument(doc);
+ }
+ }
+
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+
+ // Now search the index:
+ IndexReader r = w.getReader();
+ BitSet missing = new FixedBitSet(r.maxDoc());
+ for(int docID=0;docID<r.maxDoc();docID++) {
+ Document doc = r.document(docID);
+ if (missingSet.contains(doc.getField("id").numericValue())) {
+ missing.set(docID);
+ }
+ }
+
+ for(int iter=0;iter<100;iter++) {
+ DocIdSetIterator values = fieldCreator.iterator(r);
+ assertEquals(-1, values.docID());
+
+ while (true) {
+ int docID;
+ if (random().nextBoolean()) {
+ docID = values.nextDoc();
+ } else {
+ int range;
+ if (random().nextInt(10) == 7) {
+ // big jump
+ range = r.maxDoc()-values.docID();
+ } else {
+ // small jump
+ range = 25;
+ }
+ int inc = TestUtil.nextInt(random(), 1, range);
+ docID = values.advance(values.docID() + inc);
+ }
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ assertFalse(missing.get(docID));
+ }
+ }
+
+ IOUtils.close(r, w, directory);
+ }
+
+ protected boolean codecAcceptsHugeBinaryValues(String field) {
return true;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index 5d2e383..d56e6cb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -330,7 +330,49 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
// DocValuesFormat
try (DocValuesConsumer consumer = codec.docValuesFormat().fieldsConsumer(writeState)) {
- consumer.addNumericField(field, Collections.singleton(5));
+ consumer.addNumericField(field,
+ new EmptyDocValuesProducer() {
+ @Override
+ public NumericDocValues getNumeric(FieldInfo field) {
+ return new NumericDocValues() {
+ int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ docID++;
+ if (docID == 1) {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (docID <= 0 && target == 0) {
+ docID = 0;
+ } else {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 1;
+ }
+
+ @Override
+ public long longValue() {
+ return 5;
+ }
+ };
+ }
+ });
IOUtils.close(consumer);
IOUtils.close(consumer);
}
@@ -341,7 +383,62 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
// NormsFormat
try (NormsConsumer consumer = codec.normsFormat().normsConsumer(writeState)) {
- consumer.addNormsField(field, Collections.singleton(5));
+ consumer.addNormsField(field,
+ new NormsProducer() {
+ @Override
+ public NumericDocValues getNorms(FieldInfo field) {
+ return new NumericDocValues() {
+ int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ docID++;
+ if (docID == 1) {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (docID <= 0 && target == 0) {
+ docID = 0;
+ } else {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 1;
+ }
+
+ @Override
+ public long longValue() {
+ return 5;
+ }
+ };
+ }
+
+ @Override
+ public void checkIntegrity() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+ });
IOUtils.close(consumer);
IOUtils.close(consumer);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
index ee70fb6..dc0636d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
@@ -35,6 +35,8 @@ import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Abstract class to do basic tests for a norms format.
* NOTE: This test focuses on the norms impl, nothing else.
@@ -279,8 +281,11 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
NumericDocValues expected = r.getNumericDocValues("dv");
NumericDocValues actual = r.getNormValues("indexed");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals("doc " + i, expected.get(i), actual.get(i));
+ assertEquals(i, expected.nextDoc());
+ assertEquals(i, actual.nextDoc());
+ assertEquals("doc " + i, expected.longValue(), actual.longValue());
}
+ assertEquals(NO_MORE_DOCS, expected.nextDoc());
}
ir.close();
@@ -293,7 +298,9 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
NumericDocValues expected = r.getNumericDocValues("dv");
NumericDocValues actual = r.getNormValues("indexed");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals("doc " + i, expected.get(i), actual.get(i));
+ assertEquals(i, expected.nextDoc());
+ assertEquals(i, actual.nextDoc());
+ assertEquals("doc " + i, expected.longValue(), actual.longValue());
}
}
@@ -397,7 +404,8 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
assertNotNull(norms);
for(int i=0;i<r.maxDoc();i++) {
- assertEquals(0, norms.get(i));
+ assertEquals(i, norms.nextDoc());
+ assertEquals(0, norms.longValue());
}
r.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
index 5891df5..b981aef 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
@@ -43,6 +43,8 @@ import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Abstract class to do basic tests for a points format.
* NOTE: This test focuses on the points impl, nothing else.
@@ -167,7 +169,14 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
for(LeafReaderContext ctx : r.leaves()) {
PointValues values = ctx.reader().getPointValues();
+
NumericDocValues idValues = ctx.reader().getNumericDocValues("id");
+ int[] docIDToID = new int[ctx.reader().maxDoc()];
+ int docID;
+ while ((docID = idValues.nextDoc()) != NO_MORE_DOCS) {
+ docIDToID[docID] = (int) idValues.longValue();
+ }
+
if (values != null) {
BitSet seen = new BitSet();
values.intersect("dim",
@@ -183,7 +192,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (liveDocs.get(docID)) {
seen.set(docID);
}
- assertEquals(idValues.get(docID), NumericUtils.sortableBytesToInt(packedValue, 0));
+ assertEquals(docIDToID[docID], NumericUtils.sortableBytesToInt(packedValue, 0));
}
});
assertEquals(0, seen.cardinality());
@@ -704,6 +713,14 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
NumericDocValues idValues = MultiDocValues.getNumericValues(r, "id");
+ int[] docIDToID = new int[r.maxDoc()];
+ {
+ int docID;
+ while ((docID = idValues.nextDoc()) != NO_MORE_DOCS) {
+ docIDToID[docID] = (int) idValues.longValue();
+ }
+ }
+
Bits liveDocs = MultiFields.getLiveDocs(r);
// Verify min/max values are correct:
@@ -781,7 +798,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
@Override
public void visit(int docID) {
if (liveDocs == null || liveDocs.get(docBase+docID)) {
- hits.set((int) idValues.get(docBase+docID));
+ hits.set(docIDToID[docBase+docID]);
}
//System.out.println("visit docID=" + docID);
}
@@ -792,7 +809,6 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
return;
}
- //System.out.println("visit check docID=" + docID + " id=" + idValues.get(docID));
for(int dim=0;dim<numDims;dim++) {
//System.out.println(" dim=" + dim + " value=" + new BytesRef(packedValue, dim*numBytesPerDim, numBytesPerDim));
if (StringHelper.compare(numBytesPerDim, packedValue, dim*numBytesPerDim, queryMin[dim], 0) < 0 ||
@@ -803,7 +819,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
//System.out.println(" yes");
- hits.set((int) idValues.get(docBase+docID));
+ hits.set(docIDToID[docBase+docID]);
}
@Override
@@ -869,7 +885,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (failCount != 0) {
for(int docID=0;docID<r.maxDoc();docID++) {
- System.out.println(" docID=" + docID + " id=" + idValues.get(docID));
+ System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
}
fail(failCount + " docs failed; " + successCount + " docs succeeded");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
index adcb0af..3868b16 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
@@ -292,7 +292,8 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
final Document doc = sub.document(docID);
final Field f = (Field) doc.getField("nf");
assertTrue("got f=" + f, f instanceof StoredField);
- assertEquals(answers[(int) ids.get(docID)], f.numericValue());
+ assertEquals(docID, ids.nextDoc());
+ assertEquals(answers[(int) ids.longValue()], f.numericValue());
}
}
r.close();
[13/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValuesWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValuesWrapper.java b/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValuesWrapper.java
new file mode 100644
index 0000000..13bc207
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValuesWrapper.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Wraps a {@link LegacyBinaryDocValues} into a {@link BinaryDocValues}.
+ *
+ * @deprecated Implement {@link BinaryDocValues} directly.
+ */
+@Deprecated
+public final class LegacyBinaryDocValuesWrapper extends BinaryDocValues {
+ private final Bits docsWithField;
+ private final LegacyBinaryDocValues values;
+ private final int maxDoc;
+ private int docID = -1;
+
+ public LegacyBinaryDocValuesWrapper(Bits docsWithField, LegacyBinaryDocValues values) {
+ this.docsWithField = docsWithField;
+ this.values = values;
+ this.maxDoc = docsWithField.length();
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ docID++;
+ while (docID < maxDoc) {
+ if (docsWithField.get(docID)) {
+ return docID;
+ }
+ docID++;
+ }
+ docID = NO_MORE_DOCS;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docID) {
+ throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+ }
+ if (target == NO_MORE_DOCS) {
+ this.docID = NO_MORE_DOCS;
+ } else {
+ this.docID = target-1;
+ nextDoc();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return values.get(docID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValues.java
new file mode 100644
index 0000000..df71435
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValues.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+/**
+ * A per-document numeric value.
+ *
+ * @deprecated Use {@link NumericDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacyNumericDocValues {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LegacyNumericDocValues() {}
+
+ /**
+ * Returns the numeric value for the specified document ID.
+ * @param docID document ID to lookup
+ * @return numeric value
+ */
+ public abstract long get(int docID);
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValuesWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValuesWrapper.java b/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValuesWrapper.java
new file mode 100644
index 0000000..d9c997c
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacyNumericDocValuesWrapper.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Bits;
+
+/**
+ * Wraps a {@link LegacyNumericDocValues} into a {@link NumericDocValues}.
+ *
+ * @deprecated Implement {@link NumericDocValues} directly.
+ */
+@Deprecated
+public final class LegacyNumericDocValuesWrapper extends NumericDocValues {
+ private final Bits docsWithField;
+ private final LegacyNumericDocValues values;
+ private final int maxDoc;
+ private int docID = -1;
+
+ public LegacyNumericDocValuesWrapper(Bits docsWithField, LegacyNumericDocValues values) {
+ this.docsWithField = docsWithField;
+ this.values = values;
+ this.maxDoc = docsWithField.length();
+ }
+
+ /** Constructor used only for norms */
+ public LegacyNumericDocValuesWrapper(int maxDoc, LegacyNumericDocValues values) {
+ this(new Bits.MatchAllBits(maxDoc), values);
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ docID++;
+ while (docID < maxDoc) {
+ if (docsWithField.get(docID)) {
+ return docID;
+ }
+ docID++;
+ }
+ docID = NO_MORE_DOCS;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docID) {
+ throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+ }
+ if (target == NO_MORE_DOCS) {
+ this.docID = NO_MORE_DOCS;
+ } else {
+ this.docID = target-1;
+ nextDoc();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+
+ @Override
+ public long longValue() {
+ return values.get(docID);
+ }
+
+ @Override
+ public String toString() {
+ return "LegacyNumericDocValuesWrapper(" + values + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValues.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValues.java
new file mode 100644
index 0000000..089040f
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValues.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A per-document byte[] with presorted values.
+ * <p>
+ * Per-Document values in a SortedDocValues are deduplicated, dereferenced,
+ * and sorted into a dictionary of unique values. A pointer to the
+ * dictionary value (ordinal) can be retrieved for each document. Ordinals
+ * are dense and in increasing sorted order.
+ *
+ * @deprecated Use {@link SortedDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacySortedDocValues extends LegacyBinaryDocValues {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LegacySortedDocValues() {}
+
+ /**
+ * Returns the ordinal for the specified docID.
+ * @param docID document ID to lookup
+ * @return ordinal for the document: this is dense, starts at 0, then
+ * increments by 1 for the next value in sorted order. Note that
+ * missing values are indicated by -1.
+ */
+ public abstract int getOrd(int docID);
+
+ /** Retrieves the value for the specified ordinal. The returned
+ * {@link BytesRef} may be re-used across calls to {@link #lookupOrd(int)}
+ * so make sure to {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want
+ * to keep it around.
+ * @param ord ordinal to lookup (must be >= 0 and < {@link #getValueCount()})
+ * @see #getOrd(int)
+ */
+ public abstract BytesRef lookupOrd(int ord);
+
+ /**
+ * Returns the number of unique values.
+ * @return number of unique values in this SortedDocValues. This is
+ * also equivalent to one plus the maximum ordinal.
+ */
+ public abstract int getValueCount();
+
+ private final BytesRef empty = new BytesRef();
+
+ @Override
+ public BytesRef get(int docID) {
+ int ord = getOrd(docID);
+ if (ord == -1) {
+ return empty;
+ } else {
+ return lookupOrd(ord);
+ }
+ }
+
+ /** If {@code key} exists, returns its ordinal, else
+ * returns {@code -insertionPoint-1}, like {@code
+ * Arrays.binarySearch}.
+ *
+ * @param key Key to look up
+ **/
+ public int lookupTerm(BytesRef key) {
+ int low = 0;
+ int high = getValueCount()-1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ final BytesRef term = lookupOrd(mid);
+ int cmp = term.compareTo(key);
+
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ return mid; // key found
+ }
+ }
+
+ return -(low + 1); // key not found.
+ }
+
+ /**
+ * Returns a {@link TermsEnum} over the values.
+ * The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}.
+ */
+ public TermsEnum termsEnum() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValuesWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValuesWrapper.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValuesWrapper.java
new file mode 100644
index 0000000..d8ef2f4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedDocValuesWrapper.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Wraps a {@link LegacySortedDocValues} into a {@link SortedDocValues}.
+ *
+ * @deprecated Implement {@link SortedDocValues} directly.
+ */
+@Deprecated
+public final class LegacySortedDocValuesWrapper extends SortedDocValues {
+ private final LegacySortedDocValues values;
+ private final int maxDoc;
+ private int docID = -1;
+ private int ord;
+
+ public LegacySortedDocValuesWrapper(LegacySortedDocValues values, int maxDoc) {
+ this.values = values;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert docID != NO_MORE_DOCS;
+ docID++;
+ while (docID < maxDoc) {
+ ord = values.getOrd(docID);
+ if (ord != -1) {
+ return docID;
+ }
+ docID++;
+ }
+ docID = NO_MORE_DOCS;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docID) {
+ throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+ }
+ if (target >= maxDoc) {
+ this.docID = NO_MORE_DOCS;
+ } else {
+ this.docID = target-1;
+ nextDoc();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ return values.lookupOrd(ord);
+ }
+
+ @Override
+ public int getValueCount() {
+ return values.getValueCount();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValues.java
new file mode 100644
index 0000000..42e9a5d
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValues.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+/**
+ * A list of per-document numeric values, sorted
+ * according to {@link Long#compare(long, long)}.
+ *
+ * @deprecated Use {@link SortedNumericDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacySortedNumericDocValues {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LegacySortedNumericDocValues() {}
+
+ /**
+ * Positions to the specified document
+ */
+ public abstract void setDocument(int doc);
+
+ /**
+ * Retrieve the value for the current document at the specified index.
+ * An index ranges from {@code 0} to {@code count()-1}.
+ */
+ public abstract long valueAt(int index);
+
+ /**
+ * Retrieves the count of values for the current document.
+ * This may be zero if a document has no values.
+ */
+ public abstract int count();
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValuesWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValuesWrapper.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValuesWrapper.java
new file mode 100644
index 0000000..bf3c6cd
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedNumericDocValuesWrapper.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+/**
+ * Wraps a {@link LegacySortedNumericDocValues} into a {@link SortedNumericDocValues}.
+ *
+ * @deprecated Implement {@link SortedNumericDocValues} directly.
+ */
+@Deprecated
+public final class LegacySortedNumericDocValuesWrapper extends SortedNumericDocValues {
+ private final LegacySortedNumericDocValues values;
+ private final int maxDoc;
+ private int docID = -1;
+ private int upto;
+
+ public LegacySortedNumericDocValuesWrapper(LegacySortedNumericDocValues values, int maxDoc) {
+ this.values = values;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert docID != NO_MORE_DOCS;
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ values.setDocument(docID);
+ if (values.count() != 0) {
+ break;
+ }
+ }
+ upto = 0;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docID) {
+ throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+ }
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = target-1;
+ nextDoc();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public long nextValue() {
+ return values.valueAt(upto++);
+ }
+
+ @Override
+ public int docValueCount() {
+ return values.count();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValues.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValues.java
new file mode 100644
index 0000000..dae1179
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValues.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A per-document set of presorted byte[] values.
+ * <p>
+ * Per-Document values in a SortedDocValues are deduplicated, dereferenced,
+ * and sorted into a dictionary of unique values. A pointer to the
+ * dictionary value (ordinal) can be retrieved for each document. Ordinals
+ * are dense and in increasing sorted order.
+ *
+ * @deprecated Use {@link SortedSetDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacySortedSetDocValues {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LegacySortedSetDocValues() {}
+
+ /** When returned by {@link #nextOrd()} it means there are no more
+ * ordinals for the document.
+ */
+ public static final long NO_MORE_ORDS = -1;
+
+ /**
+ * Returns the next ordinal for the current document (previously
+ * set by {@link #setDocument(int)}.
+ * @return next ordinal for the document, or {@link #NO_MORE_ORDS}.
+ * ordinals are dense, start at 0, then increment by 1 for
+ * the next value in sorted order.
+ */
+ public abstract long nextOrd();
+
+ /**
+ * Sets iteration to the specified docID
+ * @param docID document ID
+ */
+ public abstract void setDocument(int docID);
+
+ /** Retrieves the value for the specified ordinal. The returned
+ * {@link BytesRef} may be re-used across calls to lookupOrd so make sure to
+ * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
+ * around.
+ * @param ord ordinal to lookup
+ * @see #nextOrd
+ */
+ public abstract BytesRef lookupOrd(long ord);
+
+ /**
+ * Returns the number of unique values.
+ * @return number of unique values in this SortedDocValues. This is
+ * also equivalent to one plus the maximum ordinal.
+ */
+ public abstract long getValueCount();
+
+ /** If {@code key} exists, returns its ordinal, else
+ * returns {@code -insertionPoint-1}, like {@code
+ * Arrays.binarySearch}.
+ *
+ * @param key Key to look up
+ **/
+ public long lookupTerm(BytesRef key) {
+ long low = 0;
+ long high = getValueCount()-1;
+
+ while (low <= high) {
+ long mid = (low + high) >>> 1;
+ final BytesRef term = lookupOrd(mid);
+ int cmp = term.compareTo(key);
+
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ return mid; // key found
+ }
+ }
+
+ return -(low + 1); // key not found.
+ }
+
+ /**
+ * Returns a {@link TermsEnum} over the values.
+ * The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}.
+ */
+ public TermsEnum termsEnum() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValuesWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValuesWrapper.java b/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValuesWrapper.java
new file mode 100644
index 0000000..45d12d2
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacySortedSetDocValuesWrapper.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Wraps a {@link LegacySortedSetDocValues} into a {@link SortedSetDocValues}.
+ *
+ * @deprecated Implement {@link SortedSetDocValues} directly.
+ */
+@Deprecated
+public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues {
+ private final LegacySortedSetDocValues values;
+ private final int maxDoc;
+ private int docID = -1;
+ private long ord;
+
+ public LegacySortedSetDocValuesWrapper(LegacySortedSetDocValues values, int maxDoc) {
+ this.values = values;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert docID != NO_MORE_DOCS;
+ docID++;
+ while (docID < maxDoc) {
+ values.setDocument(docID);
+ ord = values.nextOrd();
+ if (ord != NO_MORE_ORDS) {
+ return docID;
+ }
+ docID++;
+ }
+ docID = NO_MORE_DOCS;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docID) {
+ throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+ }
+ if (target >= maxDoc) {
+ this.docID = NO_MORE_DOCS;
+ } else {
+ this.docID = target-1;
+ nextDoc();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public long nextOrd() {
+ long result = ord;
+ if (result != NO_MORE_ORDS) {
+ ord = values.nextOrd();
+ }
+ return result;
+ }
+
+ @Override
+ public BytesRef lookupOrd(long ord) {
+ return values.lookupOrd((int) ord);
+ }
+
+ @Override
+ public long getValueCount() {
+ return values.getValueCount();
+ }
+
+ @Override
+ public String toString() {
+ return "LegacySortedSetDocValuesWrapper(" + values + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
index bcecf2f..4c06b38 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeReaderWrapper.java
@@ -161,21 +161,6 @@ class MergeReaderWrapper extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- ensureOpen();
- FieldInfo fi = getFieldInfos().fieldInfo(field);
- if (fi == null) {
- // Field does not exist
- return null;
- }
- if (fi.getDocValuesType() == DocValuesType.NONE) {
- // Field was not indexed with doc values
- return null;
- }
- return docValues.getDocsWithField(fi);
- }
-
- @Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
FieldInfo fi = getFieldInfos().fieldInfo(field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
index af4dcfc..4054e90 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
@@ -27,7 +27,6 @@ import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
import org.apache.lucene.index.MultiTermsEnum.TermsEnumWithSlice;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.LongValues;
@@ -73,39 +72,86 @@ public class MultiDocValues {
return null;
}
- boolean anyReal = false;
- final NumericDocValues[] values = new NumericDocValues[size];
- final int[] starts = new int[size+1];
- for (int i = 0; i < size; i++) {
- LeafReaderContext context = leaves.get(i);
- NumericDocValues v = context.reader().getNormValues(field);
- if (v == null) {
- v = DocValues.emptyNumeric();
- } else {
- anyReal = true;
+ return new NumericDocValues() {
+ private int nextLeaf;
+ private NumericDocValues currentValues;
+ private LeafReaderContext currentLeaf;
+ private int docID = -1;
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ if (currentValues == null) {
+ if (nextLeaf == leaves.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(nextLeaf);
+ currentValues = currentLeaf.reader().getNormValues(field);
+ nextLeaf++;
+ continue;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int docID() {
+ return docID;
}
- values[i] = v;
- starts[i] = context.docBase;
- }
- starts[size] = r.maxDoc();
-
- assert anyReal;
- return new NumericDocValues() {
@Override
- public long get(int docID) {
- int subIndex = ReaderUtil.subIndex(docID, starts);
- return values[subIndex].get(docID - starts[subIndex]);
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, leaves);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == leaves.size()) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(readerIndex);
+ currentValues = currentLeaf.reader().getNormValues(field);
+ if (currentValues == null) {
+ return nextDoc();
+ }
+ nextLeaf = readerIndex+1;
+ }
+ int newDocID = currentValues.advance(targetDocID - currentLeaf.docBase);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+
+ @Override
+ public long longValue() {
+ return currentValues.longValue();
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
}
};
}
- /** Returns a NumericDocValues for a reader's docvalues (potentially merging on-the-fly)
- * <p>
- * This is a slow way to access numeric values. Instead, access them per-segment
- * with {@link LeafReader#getNumericDocValues(String)}
- * </p>
- * */
+ /** Returns a NumericDocValues for a reader's docvalues (potentially merging on-the-fly) */
public static NumericDocValues getNumericValues(final IndexReader r, final String field) throws IOException {
final List<LeafReaderContext> leaves = r.leaves();
final int size = leaves.size();
@@ -116,124 +162,204 @@ public class MultiDocValues {
}
boolean anyReal = false;
- final NumericDocValues[] values = new NumericDocValues[size];
- final int[] starts = new int[size+1];
- for (int i = 0; i < size; i++) {
- LeafReaderContext context = leaves.get(i);
- NumericDocValues v = context.reader().getNumericDocValues(field);
- if (v == null) {
- v = DocValues.emptyNumeric();
- } else {
- anyReal = true;
+ for(LeafReaderContext leaf : leaves) {
+ FieldInfo fieldInfo = leaf.reader().getFieldInfos().fieldInfo(field);
+ if (fieldInfo != null) {
+ DocValuesType dvType = fieldInfo.getDocValuesType();
+ if (dvType == DocValuesType.NUMERIC) {
+ anyReal = true;
+ break;
+ }
}
- values[i] = v;
- starts[i] = context.docBase;
}
- starts[size] = r.maxDoc();
- if (!anyReal) {
+ if (anyReal == false) {
return null;
- } else {
- return new NumericDocValues() {
- @Override
- public long get(int docID) {
- int subIndex = ReaderUtil.subIndex(docID, starts);
- return values[subIndex].get(docID - starts[subIndex]);
- }
- };
- }
- }
-
- /** Returns a Bits for a reader's docsWithField (potentially merging on-the-fly)
- * <p>
- * This is a slow way to access this bitset. Instead, access them per-segment
- * with {@link LeafReader#getDocsWithField(String)}
- * </p>
- * */
- public static Bits getDocsWithField(final IndexReader r, final String field) throws IOException {
- final List<LeafReaderContext> leaves = r.leaves();
- final int size = leaves.size();
- if (size == 0) {
- return null;
- } else if (size == 1) {
- return leaves.get(0).reader().getDocsWithField(field);
}
+
- boolean anyReal = false;
- boolean anyMissing = false;
- final Bits[] values = new Bits[size];
- final int[] starts = new int[size+1];
- for (int i = 0; i < size; i++) {
- LeafReaderContext context = leaves.get(i);
- Bits v = context.reader().getDocsWithField(field);
- if (v == null) {
- v = new Bits.MatchNoBits(context.reader().maxDoc());
- anyMissing = true;
- } else {
- anyReal = true;
- if (v instanceof Bits.MatchAllBits == false) {
- anyMissing = true;
+ return new NumericDocValues() {
+ private int nextLeaf;
+ private NumericDocValues currentValues;
+ private LeafReaderContext currentLeaf;
+ private int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ while (currentValues == null) {
+ if (nextLeaf == leaves.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(nextLeaf);
+ currentValues = currentLeaf.reader().getNumericDocValues(field);
+ nextLeaf++;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, leaves);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == leaves.size()) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(readerIndex);
+ currentValues = currentLeaf.reader().getNumericDocValues(field);
+ nextLeaf = readerIndex+1;
+ if (currentValues == null) {
+ return nextDoc();
+ }
+ }
+ int newDocID = currentValues.advance(targetDocID - currentLeaf.docBase);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
}
}
- values[i] = v;
- starts[i] = context.docBase;
- }
- starts[size] = r.maxDoc();
- if (!anyReal) {
- return null;
- } else if (!anyMissing) {
- return new Bits.MatchAllBits(r.maxDoc());
- } else {
- return new MultiBits(values, starts, false);
- }
+ @Override
+ public long longValue() {
+ return currentValues.longValue();
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+ };
}
- /** Returns a BinaryDocValues for a reader's docvalues (potentially merging on-the-fly)
- * <p>
- * This is a slow way to access binary values. Instead, access them per-segment
- * with {@link LeafReader#getBinaryDocValues(String)}
- * </p>
- */
+ /** Returns a BinaryDocValues for a reader's docvalues (potentially merging on-the-fly) */
public static BinaryDocValues getBinaryValues(final IndexReader r, final String field) throws IOException {
final List<LeafReaderContext> leaves = r.leaves();
final int size = leaves.size();
-
if (size == 0) {
return null;
} else if (size == 1) {
return leaves.get(0).reader().getBinaryDocValues(field);
}
-
+
boolean anyReal = false;
- final BinaryDocValues[] values = new BinaryDocValues[size];
- final int[] starts = new int[size+1];
- for (int i = 0; i < size; i++) {
- LeafReaderContext context = leaves.get(i);
- BinaryDocValues v = context.reader().getBinaryDocValues(field);
- if (v == null) {
- v = DocValues.emptyBinary();
- } else {
- anyReal = true;
+ for(LeafReaderContext leaf : leaves) {
+ FieldInfo fieldInfo = leaf.reader().getFieldInfos().fieldInfo(field);
+ if (fieldInfo != null) {
+ DocValuesType dvType = fieldInfo.getDocValuesType();
+ if (dvType == DocValuesType.BINARY) {
+ anyReal = true;
+ break;
+ }
}
- values[i] = v;
- starts[i] = context.docBase;
}
- starts[size] = r.maxDoc();
-
- if (!anyReal) {
+
+ if (anyReal == false) {
return null;
- } else {
- return new BinaryDocValues() {
- @Override
- public BytesRef get(int docID) {
- int subIndex = ReaderUtil.subIndex(docID, starts);
- return values[subIndex].get(docID - starts[subIndex]);
- }
- };
}
+
+ return new BinaryDocValues() {
+ private int nextLeaf;
+ private BinaryDocValues currentValues;
+ private LeafReaderContext currentLeaf;
+ private int docID = -1;
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ while (currentValues == null) {
+ if (nextLeaf == leaves.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(nextLeaf);
+ currentValues = currentLeaf.reader().getBinaryDocValues(field);
+ nextLeaf++;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, leaves);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == leaves.size()) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(readerIndex);
+ currentValues = currentLeaf.reader().getBinaryDocValues(field);
+ nextLeaf = readerIndex+1;
+ if (currentValues == null) {
+ return nextDoc();
+ }
+ }
+ int newDocID = currentValues.advance(targetDocID - currentLeaf.docBase);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return currentValues.binaryValue();
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+ };
}
-
+
/** Returns a SortedNumericDocValues for a reader's docvalues (potentially merging on-the-fly)
* <p>
* This is a slow way to access sorted numeric values. Instead, access them per-segment
@@ -252,6 +378,7 @@ public class MultiDocValues {
boolean anyReal = false;
final SortedNumericDocValues[] values = new SortedNumericDocValues[size];
final int[] starts = new int[size+1];
+ long totalCost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
SortedNumericDocValues v = context.reader().getSortedNumericDocValues(field);
@@ -262,33 +389,93 @@ public class MultiDocValues {
}
values[i] = v;
starts[i] = context.docBase;
+ totalCost += v.cost();
}
starts[size] = r.maxDoc();
- if (!anyReal) {
+ if (anyReal == false) {
return null;
- } else {
- return new SortedNumericDocValues() {
- SortedNumericDocValues current;
-
- @Override
- public void setDocument(int doc) {
- int subIndex = ReaderUtil.subIndex(doc, starts);
- current = values[subIndex];
- current.setDocument(doc - starts[subIndex]);
- }
+ }
- @Override
- public long valueAt(int index) {
- return current.valueAt(index);
- }
+ final long finalTotalCost = totalCost;
+
+ return new SortedNumericDocValues() {
+ private int nextLeaf;
+ private SortedNumericDocValues currentValues;
+ private LeafReaderContext currentLeaf;
+ private int docID = -1;
- @Override
- public int count() {
- return current.count();
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ if (currentValues == null) {
+ if (nextLeaf == leaves.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(nextLeaf);
+ currentValues = values[nextLeaf];
+ nextLeaf++;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
}
- };
- }
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, leaves);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == leaves.size()) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentLeaf = leaves.get(readerIndex);
+ currentValues = values[readerIndex];
+ nextLeaf = readerIndex+1;
+ }
+ int newDocID = currentValues.advance(targetDocID - currentLeaf.docBase);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
+ } else {
+ docID = currentLeaf.docBase + newDocID;
+ return docID;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return finalTotalCost;
+ }
+
+ @Override
+ public int docValueCount() {
+ return currentValues.docValueCount();
+ }
+
+ @Override
+ public long nextValue() throws IOException {
+ return currentValues.nextValue();
+ }
+ };
}
/** Returns a SortedDocValues for a reader's docvalues (potentially doing extremely slow things).
@@ -310,6 +497,7 @@ public class MultiDocValues {
boolean anyReal = false;
final SortedDocValues[] values = new SortedDocValues[size];
final int[] starts = new int[size+1];
+ long totalCost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
SortedDocValues v = context.reader().getSortedDocValues(field);
@@ -317,17 +505,18 @@ public class MultiDocValues {
v = DocValues.emptySorted();
} else {
anyReal = true;
+ totalCost += v.cost();
}
values[i] = v;
starts[i] = context.docBase;
}
starts[size] = r.maxDoc();
- if (!anyReal) {
+ if (anyReal == false) {
return null;
} else {
OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT);
- return new MultiSortedDocValues(values, starts, mapping);
+ return new MultiSortedDocValues(values, starts, mapping, totalCost);
}
}
@@ -350,6 +539,7 @@ public class MultiDocValues {
boolean anyReal = false;
final SortedSetDocValues[] values = new SortedSetDocValues[size];
final int[] starts = new int[size+1];
+ long totalCost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
SortedSetDocValues v = context.reader().getSortedSetDocValues(field);
@@ -357,17 +547,18 @@ public class MultiDocValues {
v = DocValues.emptySortedSet();
} else {
anyReal = true;
+ totalCost += v.cost();
}
values[i] = v;
starts[i] = context.docBase;
}
starts[size] = r.maxDoc();
- if (!anyReal) {
+ if (anyReal == false) {
return null;
} else {
OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT);
- return new MultiSortedSetDocValues(values, starts, mapping);
+ return new MultiSortedSetDocValues(values, starts, mapping, totalCost);
}
}
@@ -659,20 +850,81 @@ public class MultiDocValues {
public final SortedDocValues values[];
/** ordinal map mapping ords from <code>values</code> to global ord space */
public final OrdinalMap mapping;
+ private final long totalCost;
+
+ private int nextLeaf;
+ private SortedDocValues currentValues;
+ private int currentDocStart;
+ private int docID = -1;
/** Creates a new MultiSortedDocValues over <code>values</code> */
- public MultiSortedDocValues(SortedDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
+ public MultiSortedDocValues(SortedDocValues values[], int docStarts[], OrdinalMap mapping, long totalCost) throws IOException {
assert docStarts.length == values.length + 1;
this.values = values;
this.docStarts = docStarts;
this.mapping = mapping;
+ this.totalCost = totalCost;
}
@Override
- public int getOrd(int docID) {
- int subIndex = ReaderUtil.subIndex(docID, docStarts);
- int segmentOrd = values[subIndex].getOrd(docID - docStarts[subIndex]);
- return segmentOrd == -1 ? segmentOrd : (int) mapping.getGlobalOrds(subIndex).get(segmentOrd);
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ while (currentValues == null) {
+ if (nextLeaf == values.length) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentDocStart = docStarts[nextLeaf];
+ currentValues = values[nextLeaf];
+ nextLeaf++;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentDocStart + newDocID;
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, docStarts);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == values.length) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentDocStart = docStarts[readerIndex];
+ currentValues = values[readerIndex];
+ nextLeaf = readerIndex+1;
+ }
+ int newDocID = currentValues.advance(targetDocID - currentDocStart);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
+ } else {
+ docID = currentDocStart + newDocID;
+ return docID;
+ }
+ }
+
+ @Override
+ public int ordValue() {
+ return (int) mapping.getGlobalOrds(nextLeaf-1).get(currentValues.ordValue());
}
@Override
@@ -686,6 +938,11 @@ public class MultiDocValues {
public int getValueCount() {
return (int) mapping.getValueCount();
}
+
+ @Override
+ public long cost() {
+ return totalCost;
+ }
}
/**
@@ -699,34 +956,88 @@ public class MultiDocValues {
public final SortedSetDocValues values[];
/** ordinal map mapping ords from <code>values</code> to global ord space */
public final OrdinalMap mapping;
- int currentSubIndex;
- LongValues currentGlobalOrds;
-
+ private final long totalCost;
+
+ private int nextLeaf;
+ private SortedSetDocValues currentValues;
+ private int currentDocStart;
+ private int docID = -1;
+
/** Creates a new MultiSortedSetDocValues over <code>values</code> */
- public MultiSortedSetDocValues(SortedSetDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
+ public MultiSortedSetDocValues(SortedSetDocValues values[], int docStarts[], OrdinalMap mapping, long totalCost) throws IOException {
assert docStarts.length == values.length + 1;
this.values = values;
this.docStarts = docStarts;
this.mapping = mapping;
+ this.totalCost = totalCost;
}
@Override
- public long nextOrd() {
- long segmentOrd = values[currentSubIndex].nextOrd();
- if (segmentOrd == NO_MORE_ORDS) {
- return segmentOrd;
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (true) {
+ while (currentValues == null) {
+ if (nextLeaf == values.length) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentDocStart = docStarts[nextLeaf];
+ currentValues = values[nextLeaf];
+ nextLeaf++;
+ }
+
+ int newDocID = currentValues.nextDoc();
+
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ continue;
+ } else {
+ docID = currentDocStart + newDocID;
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int targetDocID) throws IOException {
+ if (targetDocID <= docID) {
+ throw new IllegalArgumentException("can only advance beyond current document: on docID=" + docID + " but targetDocID=" + targetDocID);
+ }
+ int readerIndex = ReaderUtil.subIndex(targetDocID, docStarts);
+ if (readerIndex >= nextLeaf) {
+ if (readerIndex == values.length) {
+ currentValues = null;
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ currentDocStart = docStarts[readerIndex];
+ currentValues = values[readerIndex];
+ nextLeaf = readerIndex+1;
+ }
+ int newDocID = currentValues.advance(targetDocID - currentDocStart);
+ if (newDocID == NO_MORE_DOCS) {
+ currentValues = null;
+ return nextDoc();
} else {
- return currentGlobalOrds.get(segmentOrd);
+ docID = currentDocStart + newDocID;
+ return docID;
}
}
@Override
- public void setDocument(int docID) {
- currentSubIndex = ReaderUtil.subIndex(docID, docStarts);
- currentGlobalOrds = mapping.getGlobalOrds(currentSubIndex);
- values[currentSubIndex].setDocument(docID - docStarts[currentSubIndex]);
+ public long nextOrd() throws IOException {
+ long segmentOrd = currentValues.nextOrd();
+ if (segmentOrd == NO_MORE_ORDS) {
+ return segmentOrd;
+ } else {
+ return mapping.getGlobalOrds(nextLeaf-1).get(segmentOrd);
+ }
}
-
+
@Override
public BytesRef lookupOrd(long ord) {
int subIndex = mapping.getFirstSegmentNumber(ord);
@@ -738,5 +1049,10 @@ public class MultiDocValues {
public long getValueCount() {
return mapping.getValueCount();
}
+
+ @Override
+ public long cost() {
+ return totalCost;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index 51cabab..ee969c7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -18,18 +18,18 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.MergeState.DocMap;
-import org.apache.lucene.index.MergeState;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
+@SuppressWarnings({"unchecked","rawtypes"})
final class MultiSorter {
/** Does a merge sort of the leaves of the incoming reader, returning {@link DocMap} to map each leaf's
@@ -39,9 +39,9 @@ final class MultiSorter {
// TODO: optimize if only 1 reader is incoming, though that's a rare case
SortField fields[] = sort.getSort();
- final CrossReaderComparator[] comparators = new CrossReaderComparator[fields.length];
+ final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
for(int i=0;i<fields.length;i++) {
- comparators[i] = getComparator(readers, fields[i]);
+ comparables[i] = getComparableProviders(readers, fields[i]);
}
int leafCount = readers.size();
@@ -49,8 +49,8 @@ final class MultiSorter {
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
- for(int i=0;i<comparators.length;i++) {
- int cmp = comparators[i].compare(a.readerIndex, a.docID, b.readerIndex, b.docID);
+ for(int i=0;i<comparables.length;i++) {
+ int cmp = a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@@ -59,8 +59,9 @@ final class MultiSorter {
// tie-break by docID natural order:
if (a.readerIndex != b.readerIndex) {
return a.readerIndex < b.readerIndex;
+ } else {
+ return a.docID < b.docID;
}
- return a.docID < b.docID;
}
};
@@ -68,10 +69,16 @@ final class MultiSorter {
for(int i=0;i<leafCount;i++) {
CodecReader reader = readers.get(i);
- queue.add(new LeafAndDocID(i, reader.getLiveDocs(), reader.maxDoc()));
+ LeafAndDocID leaf = new LeafAndDocID(i, reader.getLiveDocs(), reader.maxDoc(), comparables.length);
+ for(int j=0;j<comparables.length;j++) {
+ leaf.values[j] = comparables[j][i].getComparable(leaf.docID);
+ assert leaf.values[j] != null;
+ }
+ queue.add(leaf);
builders[i] = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
}
+ // merge sort:
int mappedDocID = 0;
while (queue.size() != 0) {
LeafAndDocID top = queue.top();
@@ -81,6 +88,10 @@ final class MultiSorter {
}
top.docID++;
if (top.docID < top.maxDoc) {
+ for(int j=0;j<comparables.length;j++) {
+ top.values[j] = comparables[j][top.readerIndex].getComparable(top.docID);
+ assert top.values[j] != null;
+ }
queue.updateTop();
} else {
queue.pop();
@@ -110,32 +121,41 @@ final class MultiSorter {
final int readerIndex;
final Bits liveDocs;
final int maxDoc;
+ final Comparable[] values;
int docID;
- public LeafAndDocID(int readerIndex, Bits liveDocs, int maxDoc) {
+ public LeafAndDocID(int readerIndex, Bits liveDocs, int maxDoc, int numComparables) {
this.readerIndex = readerIndex;
this.liveDocs = liveDocs;
this.maxDoc = maxDoc;
+ this.values = new Comparable[numComparables];
}
}
- private interface CrossReaderComparator {
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB);
+ /** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
+ private interface ComparableProvider {
+ public Comparable getComparable(int docID) throws IOException;
}
- private static CrossReaderComparator getComparator(List<CodecReader> readers, SortField sortField) throws IOException {
+ /** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
+ private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
+
+ ComparableProvider[] providers = new ComparableProvider[readers.size()];
+
switch(sortField.getType()) {
case STRING:
{
// this uses the efficient segment-local ordinal map:
- MultiReader multiReader = new MultiReader(readers.toArray(new LeafReader[readers.size()]));
- final SortedDocValues sorted = MultiDocValues.getSortedValues(multiReader, sortField.getField());
- final int[] docStarts = new int[readers.size()];
- List<LeafReaderContext> leaves = multiReader.leaves();
+ final SortedDocValues[] values = new SortedDocValues[readers.size()];
for(int i=0;i<readers.size();i++) {
- docStarts[i] = leaves.get(i).docBase;
+ SortedDocValues v = readers.get(i).getSortedDocValues(sortField.getField());
+ if (v == null) {
+ v = DocValues.emptySorted();
+ }
+ values[i] = v;
}
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
missingOrd = Integer.MAX_VALUE;
@@ -150,31 +170,41 @@ final class MultiSorter {
reverseMul = 1;
}
- return new CrossReaderComparator() {
- @Override
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB) {
- int ordA = sorted.getOrd(docStarts[readerIndexA] + docIDA);
- if (ordA == -1) {
- ordA = missingOrd;
- }
- int ordB = sorted.getOrd(docStarts[readerIndexB] + docIDB);
- if (ordB == -1) {
- ordB = missingOrd;
- }
- return reverseMul * Integer.compare(ordA, ordB);
- }
- };
+ for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
+ final SortedDocValues readerValues = values[readerIndex];
+ final LongValues globalOrds = ordinalMap.getGlobalOrds(readerIndex);
+ providers[readerIndex] = new ComparableProvider() {
+ // used only by assert:
+ int lastDocID = -1;
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs must be sent in order, but lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
+
+ @Override
+ public Comparable getComparable(int docID) throws IOException {
+ assert docsInOrder(docID);
+ int readerDocID = readerValues.docID();
+ if (readerDocID < docID) {
+ readerDocID = readerValues.advance(docID);
+ }
+ if (readerDocID == docID) {
+ // translate segment's ord to global ord space:
+ return reverseMul * (int) globalOrds.get(readerValues.ordValue());
+ } else {
+ return missingOrd;
+ }
+ }
+ };
+ }
}
+ break;
case LONG:
{
- List<NumericDocValues> values = new ArrayList<>();
- List<Bits> docsWithFields = new ArrayList<>();
- for(CodecReader reader : readers) {
- values.add(DocValues.getNumeric(reader, sortField.getField()));
- docsWithFields.add(DocValues.getDocsWithField(reader, sortField.getField()));
- }
-
final int reverseMul;
if (sortField.getReverse()) {
reverseMul = -1;
@@ -182,44 +212,47 @@ final class MultiSorter {
reverseMul = 1;
}
- final long missingValue;
-
+ final Long missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Long) sortField.getMissingValue();
} else {
- missingValue = 0;
+ missingValue = 0L;
}
- return new CrossReaderComparator() {
- @Override
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB) {
- long valueA;
- if (docsWithFields.get(readerIndexA).get(docIDA)) {
- valueA = values.get(readerIndexA).get(docIDA);
- } else {
- valueA = missingValue;
- }
-
- long valueB;
- if (docsWithFields.get(readerIndexB).get(docIDB)) {
- valueB = values.get(readerIndexB).get(docIDB);
- } else {
- valueB = missingValue;
- }
- return reverseMul * Long.compare(valueA, valueB);
- }
- };
+ for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
+ final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+
+ providers[readerIndex] = new ComparableProvider() {
+ // used only by assert:
+ int lastDocID = -1;
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs must be sent in order, but lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
+
+ @Override
+ public Comparable getComparable(int docID) throws IOException {
+ assert docsInOrder(docID);
+ int readerDocID = values.docID();
+ if (readerDocID < docID) {
+ readerDocID = values.advance(docID);
+ }
+ if (readerDocID == docID) {
+ return reverseMul * values.longValue();
+ } else {
+ return missingValue;
+ }
+ }
+ };
+ }
}
+ break;
case INT:
{
- List<NumericDocValues> values = new ArrayList<>();
- List<Bits> docsWithFields = new ArrayList<>();
- for(CodecReader reader : readers) {
- values.add(DocValues.getNumeric(reader, sortField.getField()));
- docsWithFields.add(DocValues.getDocsWithField(reader, sortField.getField()));
- }
-
final int reverseMul;
if (sortField.getReverse()) {
reverseMul = -1;
@@ -227,44 +260,47 @@ final class MultiSorter {
reverseMul = 1;
}
- final int missingValue;
-
+ final Integer missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Integer) sortField.getMissingValue();
} else {
missingValue = 0;
}
- return new CrossReaderComparator() {
- @Override
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB) {
- int valueA;
- if (docsWithFields.get(readerIndexA).get(docIDA)) {
- valueA = (int) values.get(readerIndexA).get(docIDA);
- } else {
- valueA = missingValue;
- }
-
- int valueB;
- if (docsWithFields.get(readerIndexB).get(docIDB)) {
- valueB = (int) values.get(readerIndexB).get(docIDB);
- } else {
- valueB = missingValue;
- }
- return reverseMul * Integer.compare(valueA, valueB);
- }
- };
+ for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
+ final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+
+ providers[readerIndex] = new ComparableProvider() {
+ // used only by assert:
+ int lastDocID = -1;
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs must be sent in order, but lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
+
+ @Override
+ public Comparable getComparable(int docID) throws IOException {
+ assert docsInOrder(docID);
+ int readerDocID = values.docID();
+ if (readerDocID < docID) {
+ readerDocID = values.advance(docID);
+ }
+ if (readerDocID == docID) {
+ return reverseMul * (int) values.longValue();
+ } else {
+ return missingValue;
+ }
+ }
+ };
+ }
}
+ break;
case DOUBLE:
{
- List<NumericDocValues> values = new ArrayList<>();
- List<Bits> docsWithFields = new ArrayList<>();
- for(CodecReader reader : readers) {
- values.add(DocValues.getNumeric(reader, sortField.getField()));
- docsWithFields.add(DocValues.getDocsWithField(reader, sortField.getField()));
- }
-
final int reverseMul;
if (sortField.getReverse()) {
reverseMul = -1;
@@ -272,44 +308,47 @@ final class MultiSorter {
reverseMul = 1;
}
- final double missingValue;
-
+ final Double missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Double) sortField.getMissingValue();
} else {
missingValue = 0.0;
}
- return new CrossReaderComparator() {
- @Override
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB) {
- double valueA;
- if (docsWithFields.get(readerIndexA).get(docIDA)) {
- valueA = Double.longBitsToDouble(values.get(readerIndexA).get(docIDA));
- } else {
- valueA = missingValue;
- }
-
- double valueB;
- if (docsWithFields.get(readerIndexB).get(docIDB)) {
- valueB = Double.longBitsToDouble(values.get(readerIndexB).get(docIDB));
- } else {
- valueB = missingValue;
- }
- return reverseMul * Double.compare(valueA, valueB);
- }
- };
+ for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
+ final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+
+ providers[readerIndex] = new ComparableProvider() {
+ // used only by assert:
+ int lastDocID = -1;
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs must be sent in order, but lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
+
+ @Override
+ public Comparable getComparable(int docID) throws IOException {
+ assert docsInOrder(docID);
+ int readerDocID = values.docID();
+ if (readerDocID < docID) {
+ readerDocID = values.advance(docID);
+ }
+ if (readerDocID == docID) {
+ return reverseMul * Double.longBitsToDouble(values.longValue());
+ } else {
+ return missingValue;
+ }
+ }
+ };
+ }
}
+ break;
case FLOAT:
{
- List<NumericDocValues> values = new ArrayList<>();
- List<Bits> docsWithFields = new ArrayList<>();
- for(CodecReader reader : readers) {
- values.add(DocValues.getNumeric(reader, sortField.getField()));
- docsWithFields.add(DocValues.getDocsWithField(reader, sortField.getField()));
- }
-
final int reverseMul;
if (sortField.getReverse()) {
reverseMul = -1;
@@ -317,37 +356,49 @@ final class MultiSorter {
reverseMul = 1;
}
- final float missingValue;
-
+ final Float missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Float) sortField.getMissingValue();
} else {
missingValue = 0.0f;
}
- return new CrossReaderComparator() {
- @Override
- public int compare(int readerIndexA, int docIDA, int readerIndexB, int docIDB) {
- float valueA;
- if (docsWithFields.get(readerIndexA).get(docIDA)) {
- valueA = Float.intBitsToFloat((int) values.get(readerIndexA).get(docIDA));
- } else {
- valueA = missingValue;
- }
-
- float valueB;
- if (docsWithFields.get(readerIndexB).get(docIDB)) {
- valueB = Float.intBitsToFloat((int) values.get(readerIndexB).get(docIDB));
- } else {
- valueB = missingValue;
- }
- return reverseMul * Float.compare(valueA, valueB);
- }
- };
+ for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
+ final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+
+ providers[readerIndex] = new ComparableProvider() {
+ // used only by assert:
+ int lastDocID = -1;
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs must be sent in order, but lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
+
+ @Override
+ public Comparable getComparable(int docID) throws IOException {
+ assert docsInOrder(docID);
+ int readerDocID = values.docID();
+ if (readerDocID < docID) {
+ readerDocID = values.advance(docID);
+ }
+ if (readerDocID == docID) {
+ return reverseMul * Float.intBitsToFloat((int) values.longValue());
+ } else {
+ return missingValue;
+ }
+ }
+ };
+ }
}
+ break;
default:
throw new IllegalArgumentException("unhandled SortField.getType()=" + sortField.getType());
}
+
+ return providers;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java
index a9797a1..c444661 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java
@@ -18,10 +18,9 @@ package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.NormsConsumer;
+import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
@@ -70,51 +69,79 @@ class NormValuesWriter {
final PackedLongValues values = pending.build();
normsConsumer.addNormsField(fieldInfo,
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new NumericIterator(maxDoc, values);
- }
+ new NormsProducer() {
+ @Override
+ public NumericDocValues getNorms(FieldInfo fieldInfo2) {
+ if (fieldInfo != NormValuesWriter.this.fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedNorms(maxDoc, values);
+ }
+
+ @Override
+ public void checkIntegrity() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
});
}
+ // TODO: norms should only visit docs that had a field indexed!!
+
// iterates over the values we have in ram
- private static class NumericIterator implements Iterator<Number> {
+ private static class BufferedNorms extends NumericDocValues {
final PackedLongValues.Iterator iter;
final int size;
final int maxDoc;
- int upto;
+ private int docID = -1;
+ private long value;
- NumericIterator(int maxDoc, PackedLongValues values) {
+ BufferedNorms(int maxDoc, PackedLongValues values) {
this.maxDoc = maxDoc;
this.iter = values.iterator();
this.size = (int) values.size();
}
@Override
- public boolean hasNext() {
- return upto < maxDoc;
+ public int docID() {
+ return docID;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
+ public int nextDoc() {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
}
- Long value;
- if (upto < size) {
+ if (docID < size) {
value = iter.next();
} else {
value = MISSING;
}
- upto++;
- return value;
+ return docID;
}
-
+
@Override
- public void remove() {
+ public int advance(int target) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long cost() {
+ return maxDoc;
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
index 33ab33b..d40f56a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
@@ -14,22 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.lucene.index;
+import org.apache.lucene.search.DocIdSetIterator;
/**
* A per-document numeric value.
*/
-public abstract class NumericDocValues {
+public abstract class NumericDocValues extends DocIdSetIterator {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
protected NumericDocValues() {}
/**
- * Returns the numeric value for the specified document ID.
- * @param docID document ID to lookup
+ * Returns the numeric value for the current document ID.
* @return numeric value
*/
- public abstract long get(int docID);
+ public abstract long longValue();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 1d28611..4dd3cd0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -31,7 +31,8 @@ import org.apache.lucene.util.packed.PagedMutable;
* @lucene.experimental
*/
class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
-
+
+ // TODO: can't this just be NumericDocValues now? avoid boxing the long value...
final static class Iterator extends DocValuesFieldUpdates.Iterator {
private final int size;
private final PagedGrowableWriter values;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
index 917af66..03fbe10 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.Counter;
@@ -88,57 +86,70 @@ class NumericDocValuesWriter extends DocValuesWriter {
final PackedLongValues values = pending.build();
dvConsumer.addNumericField(fieldInfo,
- new Iterable<Number>() {
+ new EmptyDocValuesProducer() {
@Override
- public Iterator<Number> iterator() {
- return new NumericIterator(maxDoc, values, docsWithField);
+ public NumericDocValues getNumeric(FieldInfo fieldInfo) {
+ if (fieldInfo != NumericDocValuesWriter.this.fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedNumericDocValues(maxDoc, values, docsWithField);
}
});
}
// iterates over the values we have in ram
- private static class NumericIterator implements Iterator<Number> {
+ private static class BufferedNumericDocValues extends NumericDocValues {
final PackedLongValues.Iterator iter;
final FixedBitSet docsWithField;
final int size;
final int maxDoc;
- int upto;
+ private long value;
+ private int docID = -1;
- NumericIterator(int maxDoc, PackedLongValues values, FixedBitSet docsWithFields) {
+ BufferedNumericDocValues(int maxDoc, PackedLongValues values, FixedBitSet docsWithFields) {
this.maxDoc = maxDoc;
this.iter = values.iterator();
this.size = (int) values.size();
this.docsWithField = docsWithFields;
}
-
+
@Override
- public boolean hasNext() {
- return upto < maxDoc;
+ public int docID() {
+ return docID;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- Long value;
- if (upto < size) {
- long v = iter.next();
- if (docsWithField.get(upto)) {
- value = v;
+ public int nextDoc() {
+ if (docID == size-1) {
+ docID = NO_MORE_DOCS;
+ } else {
+ int next = docsWithField.nextSetBit(docID+1);
+ if (next == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
} else {
- value = null;
+ // skip missing values:
+ while (docID < next) {
+ docID++;
+ value = iter.next();
+ }
}
- } else {
- value = null;
}
- upto++;
- return value;
+ return docID;
}
-
+
@Override
- public void remove() {
+ public int advance(int target) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long cost() {
+ return docsWithField.cardinality();
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index d85ff2d..f4afa11 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -290,7 +290,7 @@ public class ParallelLeafReader extends LeafReader {
LeafReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getBinaryDocValues(field);
}
-
+
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
ensureOpen();
@@ -313,13 +313,6 @@ public class ParallelLeafReader extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- ensureOpen();
- LeafReader reader = fieldToReader.get(field);
- return reader == null ? null : reader.getDocsWithField(field);
- }
-
- @Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
LeafReader reader = fieldToReader.get(field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/RandomAccessOrds.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/RandomAccessOrds.java b/lucene/core/src/java/org/apache/lucene/index/RandomAccessOrds.java
deleted file mode 100644
index d0494f5..0000000
--- a/lucene/core/src/java/org/apache/lucene/index/RandomAccessOrds.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.index;
-
-
-/**
- * Extension of {@link SortedSetDocValues} that supports random access
- * to the ordinals of a document.
- * <p>
- * Operations via this API are independent of the iterator api ({@link #nextOrd()})
- * and do not impact its state.
- * <p>
- * Codecs can optionally extend this API if they support constant-time access
- * to ordinals for the document.
- */
-public abstract class RandomAccessOrds extends SortedSetDocValues {
-
- /** Sole constructor. (For invocation by subclass
- * constructors, typically implicit.) */
- protected RandomAccessOrds() {}
-
- /**
- * Retrieve the ordinal for the current document (previously
- * set by {@link #setDocument(int)} at the specified index.
- * <p>
- * An index ranges from {@code 0} to {@code cardinality()-1}.
- * The first ordinal value is at index {@code 0}, the next at index {@code 1},
- * and so on, as for array indexing.
- * @param index index of the ordinal for the document.
- * @return ordinal for the document at the specified index.
- */
- public abstract long ordAt(int index);
-
- /**
- * Returns the cardinality for the current document (previously
- * set by {@link #setDocument(int)}.
- */
- public abstract int cardinality();
-}
[10/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
index 0dc654c..7901214 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
@@ -23,7 +23,9 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -32,10 +34,11 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
-import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
@@ -722,9 +725,10 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
assertNotNull("oldSchemaGen=" + oldSchemaGen, oldValues);
for(int i=0;i<maxDoc;i++) {
// TODO: is this still O(blockSize^2)?
+ assertEquals(i, oldValues.nextDoc());
Document oldDoc = reader.document(i);
Document newDoc = new Document();
- newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, oldValues.get(i)));
+ newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, oldValues.longValue()));
w.addDocument(newDoc);
}
}
@@ -752,11 +756,12 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
for(int i=0;i<maxDoc;i++) {
Document oldDoc = r.document(i);
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
- if (value != numbers.get(i)) {
- if (DEBUG) System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i) + " numbers=" + numbers);
+ assertEquals(i, numbers.nextDoc());
+ if (value != numbers.longValue()) {
+ if (DEBUG) System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue() + " numbers=" + numbers);
failed = true;
} else if (failed) {
- if (DEBUG) System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i));
+ if (DEBUG) System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue());
}
}
assertFalse("FAILED field=" + fieldName + " r=" + r, failed);
@@ -817,7 +822,8 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
// TODO: is this still O(blockSize^2)?
Document oldDoc = reader.document(i);
Document newDoc = new Document();
- newDoc.add(new NumericDocValuesField("number", newSchemaGen*(oldValues.get(i)/oldSchemaGen)));
+ assertEquals(i, oldValues.nextDoc());
+ newDoc.add(new NumericDocValuesField("number", newSchemaGen*(oldValues.longValue()/oldSchemaGen)));
w.addDocument(newDoc);
}
}
@@ -850,11 +856,12 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
Document oldDoc = r.document(i);
long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
value *= schemaGen;
- if (value != numbers.get(i)) {
- System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i) + " numbers=" + numbers);
+ assertEquals(i, numbers.nextDoc());
+ if (value != numbers.longValue()) {
+ System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue() + " numbers=" + numbers);
failed = true;
} else if (failed) {
- System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i));
+ System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue());
}
}
assertFalse("FAILED r=" + r, failed);
@@ -1110,7 +1117,8 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
for(int i=0;i<maxDoc;i++) {
Document doc = leaf.document(i);
long value = Long.parseLong(doc.get("text").split(" ")[1]);
- long dvValue = numbers.get(i);
+ assertEquals(i, numbers.nextDoc());
+ long dvValue = numbers.longValue();
if (value == 0) {
assertEquals(0, dvValue);
} else {
@@ -1278,11 +1286,12 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
for(int i=0;i<maxDoc;i++) {
Document oldDoc = r.document(i);
long value = multiplier * Long.parseLong(oldDoc.get("text").split(" ")[1]);
- if (value != numbers.get(i)) {
- System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i) + " numbers=" + numbers);
+ assertEquals(i, numbers.nextDoc());
+ if (value != numbers.longValue()) {
+ System.out.println("FAIL: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue() + " numbers=" + numbers);
failed = true;
} else if (failed) {
- System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.get(i));
+ System.out.println("OK: docID=" + i + " " + oldDoc+ " value=" + value + " number=" + numbers.longValue());
}
}
if (failed) {
@@ -1304,18 +1313,16 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
private static void testNumericDVSort(IndexSearcher s) throws IOException {
// Confirm we can sort by the new DV field:
TopDocs hits = s.search(new MatchAllDocsQuery(), 100, new Sort(new SortField("number", SortField.Type.LONG)));
- NumericDocValues numbers = MultiDocValues.getNumericValues(s.getIndexReader(), "number");
long last = Long.MIN_VALUE;
for(ScoreDoc scoreDoc : hits.scoreDocs) {
long value = Long.parseLong(s.doc(scoreDoc.doc).get("text").split(" ")[1]);
assertTrue(value >= last);
- assertEquals(value, numbers.get(scoreDoc.doc));
+ assertEquals(value, ((Long) ((FieldDoc) scoreDoc).fields[0]).longValue());
last = value;
}
}
private static void testPointRangeQuery(IndexSearcher s) throws IOException {
- NumericDocValues numbers = MultiDocValues.getNumericValues(s.getIndexReader(), "number");
for(int i=0;i<100;i++) {
// Confirm we can range search by the new indexed (numeric) field:
long min = random().nextLong();
@@ -1331,7 +1338,24 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
long value = Long.parseLong(s.doc(scoreDoc.doc).get("text").split(" ")[1]);
assertTrue(value >= min);
assertTrue(value <= max);
- assertEquals(value, numbers.get(scoreDoc.doc));
+ }
+
+ Arrays.sort(hits.scoreDocs,
+ new Comparator<ScoreDoc>() {
+ @Override
+ public int compare(ScoreDoc a, ScoreDoc b) {
+ return a.doc - b.doc;
+ }
+ });
+
+ NumericDocValues numbers = MultiDocValues.getNumericValues(s.getIndexReader(), "number");
+ for(ScoreDoc hit : hits.scoreDocs) {
+ if (numbers.docID() < hit.doc) {
+ numbers.advance(hit.doc);
+ }
+ assertEquals(hit.doc, numbers.docID());
+ long value = Long.parseLong(s.doc(hit.doc).get("text").split(" ")[1]);
+ assertEquals(value, numbers.longValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 8c7090e..9ac719e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -46,6 +46,8 @@ import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Assume;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestDirectoryReader extends LuceneTestCase {
public void testDocument() throws IOException {
@@ -573,8 +575,13 @@ public class TestDirectoryReader extends LuceneTestCase {
NumericDocValues norms2 = MultiDocValues.getNormValues(index2, curField);
if (norms1 != null && norms2 != null) {
// todo: generalize this (like TestDuelingCodecs assert)
- for (int i = 0; i < index1.maxDoc(); i++) {
- assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1.get(i), norms2.get(i));
+ while (true) {
+ int docID = norms1.nextDoc();
+ assertEquals(docID, norms2.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ assertEquals("Norm different for doc " + docID + " and field '" + curField + "'.", norms1.longValue(), norms2.longValue());
}
} else {
assertNull(norms1);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 0138b90..f415381 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -204,6 +204,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
final Directory dir = newDirectory();
// NOTE: this also controls the number of threads!
final int n = TestUtil.nextInt(random(), 20, 40);
+
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
for (int i = 0; i < n; i++) {
writer.addDocument(createDocument(i, 3));
@@ -901,9 +902,14 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
// sharing same core
assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
-
- assertEquals(1, getOnlyLeafReader(oldest).getNumericDocValues("dv").get(0));
- assertEquals(2, getOnlyLeafReader(latest).getNumericDocValues("dv").get(0));
+
+ NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+
+ values = getOnlyLeafReader(latest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(2, values.longValue());
latest.close();
oldest.close();
@@ -943,9 +949,14 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
// sharing same core
assertSame(latest.leaves().get(0).reader().getCoreCacheKey(), oldest.leaves().get(0).reader().getCoreCacheKey());
-
- assertEquals(1, getOnlyLeafReader(oldest).getNumericDocValues("dv").get(0));
- assertEquals(2, getOnlyLeafReader(latest).getNumericDocValues("dv").get(0));
+
+ NumericDocValues values = getOnlyLeafReader(oldest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+
+ values = getOnlyLeafReader(latest).getNumericDocValues("dv");
+ assertEquals(0, values.nextDoc());
+ assertEquals(2, values.longValue());
latest.close();
oldest.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java
index 2266caf..1f7ef30 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java
@@ -49,7 +49,6 @@ public class TestDocValues extends LuceneTestCase {
assertNotNull(DocValues.getSorted(r, "bogus"));
assertNotNull(DocValues.getSortedSet(r, "bogus"));
assertNotNull(DocValues.getSortedNumeric(r, "bogus"));
- assertNotNull(DocValues.getDocsWithField(r, "bogus"));
dr.close();
iw.close();
@@ -84,9 +83,6 @@ public class TestDocValues extends LuceneTestCase {
expectThrows(IllegalStateException.class, () -> {
DocValues.getSortedNumeric(r, "foo");
});
- expectThrows(IllegalStateException.class, () -> {
- DocValues.getDocsWithField(r, "foo");
- });
dr.close();
iw.close();
@@ -108,7 +104,6 @@ public class TestDocValues extends LuceneTestCase {
// ok
assertNotNull(DocValues.getNumeric(r, "foo"));
assertNotNull(DocValues.getSortedNumeric(r, "foo"));
- assertNotNull(DocValues.getDocsWithField(r, "foo"));
// errors
expectThrows(IllegalStateException.class, () -> {
@@ -140,7 +135,6 @@ public class TestDocValues extends LuceneTestCase {
// ok
assertNotNull(DocValues.getBinary(r, "foo"));
- assertNotNull(DocValues.getDocsWithField(r, "foo"));
// errors
expectThrows(IllegalStateException.class, () -> {
@@ -177,7 +171,6 @@ public class TestDocValues extends LuceneTestCase {
assertNotNull(DocValues.getBinary(r, "foo"));
assertNotNull(DocValues.getSorted(r, "foo"));
assertNotNull(DocValues.getSortedSet(r, "foo"));
- assertNotNull(DocValues.getDocsWithField(r, "foo"));
// errors
expectThrows(IllegalStateException.class, () -> {
@@ -206,7 +199,6 @@ public class TestDocValues extends LuceneTestCase {
// ok
assertNotNull(DocValues.getSortedSet(r, "foo"));
- assertNotNull(DocValues.getDocsWithField(r, "foo"));
// errors
expectThrows(IllegalStateException.class, () -> {
@@ -241,11 +233,10 @@ public class TestDocValues extends LuceneTestCase {
// ok
assertNotNull(DocValues.getSortedNumeric(r, "foo"));
- assertNotNull(DocValues.getDocsWithField(r, "foo"));
// errors
expectThrows(IllegalStateException.class, () -> {
- DocValues.getBinary(r, "foo");
+ DocValues.getBinary(r, "foo");
});
expectThrows(IllegalStateException.class, () -> {
DocValues.getNumeric(r, "foo");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
index 2e0cbd9..b3260d0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
@@ -35,7 +35,6 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -109,7 +108,9 @@ public class TestDocValuesIndexing extends LuceneTestCase {
DirectoryReader r = w.getReader();
w.close();
- assertEquals(17, DocValues.getNumeric(getOnlyLeafReader(r), "field").get(0));
+ NumericDocValues values = DocValues.getNumeric(getOnlyLeafReader(r), "field");
+ assertEquals(0, values.nextDoc());
+ assertEquals(17, values.longValue());
r.close();
d.close();
}
@@ -130,7 +131,9 @@ public class TestDocValuesIndexing extends LuceneTestCase {
DirectoryReader r = w.getReader();
w.close();
- assertEquals(17, DocValues.getNumeric(getOnlyLeafReader(r), "field").get(0));
+ NumericDocValues values = DocValues.getNumeric(getOnlyLeafReader(r), "field");
+ assertEquals(0, values.nextDoc());
+ assertEquals(17, values.longValue());
r.close();
d.close();
}
@@ -150,7 +153,9 @@ public class TestDocValuesIndexing extends LuceneTestCase {
});
DirectoryReader r = w.getReader();
- assertEquals(17, getOnlyLeafReader(r).getNumericDocValues("field").get(0));
+ NumericDocValues values = DocValues.getNumeric(getOnlyLeafReader(r), "field");
+ assertEquals(0, values.nextDoc());
+ assertEquals(17, values.longValue());
r.close();
w.close();
d.close();
@@ -171,14 +176,15 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
- BinaryDocValues s = DocValues.getSorted(getOnlyLeafReader(r), "field");
-
- BytesRef bytes1 = s.get(0);
+ BinaryDocValues s = DocValues.getBinary(getOnlyLeafReader(r), "field");
+ assertEquals(0, s.nextDoc());
+ BytesRef bytes1 = s.binaryValue();
assertEquals(bytes.length, bytes1.length);
bytes[0] = 0;
assertEquals(b, bytes1);
- bytes1 = s.get(1);
+ assertEquals(1, s.nextDoc());
+ bytes1 = s.binaryValue();
assertEquals(bytes.length, bytes1.length);
bytes[0] = 1;
assertEquals(b, bytes1);
@@ -204,7 +210,8 @@ public class TestDocValuesIndexing extends LuceneTestCase {
assertTrue(dvInfo.getDocValuesType() != DocValuesType.NONE);
NumericDocValues dv = MultiDocValues.getNumericValues(r, "dv");
for (int i = 0; i < 50; i++) {
- assertEquals(i, dv.get(i));
+ assertEquals(i, dv.nextDoc());
+ assertEquals(i, dv.longValue());
Document d = r.document(i);
// cannot use d.get("dv") due to another bug!
assertNull(d.getField("dv"));
@@ -815,9 +822,6 @@ public class TestDocValuesIndexing extends LuceneTestCase {
LeafReader subR = r.leaves().get(0).reader();
assertEquals(2, subR.numDocs());
- Bits bits = DocValues.getDocsWithField(subR, "dv");
- assertTrue(bits.get(0));
- assertTrue(bits.get(1));
r.close();
dir.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 5045396..3e72a7e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -67,7 +67,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
@@ -75,6 +74,8 @@ import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestIndexSorting extends LuceneTestCase {
public void testBasicString() throws Exception {
@@ -103,9 +104,12 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
- assertEquals("aaa", values.get(0).utf8ToString());
- assertEquals("mmm", values.get(1).utf8ToString());
- assertEquals("zzz", values.get(2).utf8ToString());
+ assertEquals(0, values.nextDoc());
+ assertEquals("aaa", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
r.close();
w.close();
dir.close();
@@ -138,9 +142,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
- assertEquals(-1, values.getOrd(0));
- assertEquals("mmm", values.get(1).utf8ToString());
- assertEquals("zzz", values.get(2).utf8ToString());
+ // docID 0 is missing:
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
r.close();
w.close();
dir.close();
@@ -173,9 +179,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
- assertEquals("mmm", values.get(0).utf8ToString());
- assertEquals("zzz", values.get(1).utf8ToString());
- assertEquals(-1, values.getOrd(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
@@ -207,9 +215,12 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(-1, values.get(0));
- assertEquals(7, values.get(1));
- assertEquals(18, values.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
r.close();
w.close();
dir.close();
@@ -242,11 +253,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(0, values.get(0));
- assertFalse(docsWithField.get(0));
- assertEquals(7, values.get(1));
- assertEquals(18, values.get(2));
+ // docID 0 has no value
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
r.close();
w.close();
dir.close();
@@ -279,11 +290,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(7, values.get(0));
- assertEquals(18, values.get(1));
- assertEquals(0, values.get(2));
- assertFalse(docsWithField.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
@@ -315,9 +326,12 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(-1, values.get(0));
- assertEquals(7, values.get(1));
- assertEquals(18, values.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
r.close();
w.close();
dir.close();
@@ -350,11 +364,10 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(0, values.get(0));
- assertFalse(docsWithField.get(0));
- assertEquals(7, values.get(1));
- assertEquals(18, values.get(2));
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
r.close();
w.close();
dir.close();
@@ -387,11 +400,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(7, values.get(0));
- assertEquals(18, values.get(1));
- assertEquals(0, values.get(2));
- assertFalse(docsWithField.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
@@ -423,9 +436,12 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(-1.0, Double.longBitsToDouble(values.get(0)), 0.0);
- assertEquals(7.0, Double.longBitsToDouble(values.get(1)), 0.0);
- assertEquals(18.0, Double.longBitsToDouble(values.get(2)), 0.0);
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
r.close();
w.close();
dir.close();
@@ -458,11 +474,10 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(0.0, Double.longBitsToDouble(values.get(0)), 0.0);
- assertFalse(docsWithField.get(0));
- assertEquals(7.0, Double.longBitsToDouble(values.get(1)), 0.0);
- assertEquals(18.0, Double.longBitsToDouble(values.get(2)), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
r.close();
w.close();
dir.close();
@@ -495,11 +510,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(7.0, Double.longBitsToDouble(values.get(0)), 0.0);
- assertEquals(18.0, Double.longBitsToDouble(values.get(1)), 0.0);
- assertEquals(0.0, Double.longBitsToDouble(values.get(2)), 0.0);
- assertFalse(docsWithField.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
@@ -531,9 +546,12 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(-1.0f, Float.intBitsToFloat((int) values.get(0)), 0.0f);
- assertEquals(7.0f, Float.intBitsToFloat((int) values.get(1)), 0.0f);
- assertEquals(18.0f, Float.intBitsToFloat((int) values.get(2)), 0.0f);
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
r.close();
w.close();
dir.close();
@@ -566,11 +584,10 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(0.0f, Float.intBitsToFloat((int) values.get(0)), 0.0f);
- assertFalse(docsWithField.get(0));
- assertEquals(7.0f, Float.intBitsToFloat((int) values.get(1)), 0.0f);
- assertEquals(18.0f, Float.intBitsToFloat((int) values.get(2)), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
r.close();
w.close();
dir.close();
@@ -603,11 +620,11 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- Bits docsWithField = leaf.getDocsWithField("foo");
- assertEquals(7.0f, Float.intBitsToFloat((int) values.get(0)), 0.0f);
- assertEquals(18.0f, Float.intBitsToFloat((int) values.get(1)), 0.0f);
- assertEquals(0.0f, Float.intBitsToFloat((int) values.get(2)), 0.0f);
- assertFalse(docsWithField.get(2));
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
@@ -653,7 +670,8 @@ public class TestIndexSorting extends LuceneTestCase {
final NumericDocValues values = leaf.getNumericDocValues("foo");
long previous = Long.MIN_VALUE;
for (int i = 0; i < leaf.maxDoc(); ++i) {
- final long value = values.get(i);
+ assertEquals(i, values.nextDoc());
+ final long value = values.longValue();
assertTrue(value >= previous);
previous = value;
}
@@ -672,7 +690,9 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(0, topDocs.totalHits);
} else {
assertEquals(1, topDocs.totalHits);
- assertEquals(i, MultiDocValues.getNumericValues(reader, "id").get(topDocs.scoreDocs[0].doc));
+ NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
+ assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
+ assertEquals(i, values.longValue());
Document document = reader.document(topDocs.scoreDocs[0].doc);
assertEquals(Integer.toString(i), document.get("id"));
}
@@ -746,6 +766,7 @@ public class TestIndexSorting extends LuceneTestCase {
final int numDocs = atLeast(100);
Thread[] threads = new Thread[2];
+
final AtomicInteger updateCount = new AtomicInteger(atLeast(1000));
final CountDownLatch latch = new CountDownLatch(1);
for (int i = 0; i < threads.length; ++i) {
@@ -768,7 +789,10 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(0, topDocs.totalHits);
} else {
assertEquals(1, topDocs.totalHits);
- assertEquals(values.get(i).longValue(), MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc));
+ NumericDocValues dvs = MultiDocValues.getNumericValues(reader, "foo");
+ int docID = topDocs.scoreDocs[0].doc;
+ assertEquals(docID, dvs.advance(docID));
+ assertEquals(values.get(i).longValue(), dvs.longValue());
}
}
reader.close();
@@ -862,7 +886,10 @@ public class TestIndexSorting extends LuceneTestCase {
for (int i = 0; i < numDocs; ++i) {
final TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(i))), 1);
assertEquals(1, topDocs.totalHits);
- assertEquals(values.get(i).longValue(), MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc));
+ NumericDocValues dvs = MultiDocValues.getNumericValues(reader, "foo");
+ int hitDoc = topDocs.scoreDocs[0].doc;
+ assertEquals(hitDoc, dvs.advance(hitDoc));
+ assertEquals(values.get(i).longValue(), dvs.longValue());
}
reader.close();
w.close();
@@ -918,9 +945,15 @@ public class TestIndexSorting extends LuceneTestCase {
final TopDocs topDocs2 = searcher2.search(query, 1);
assertEquals(topDocs.totalHits, topDocs2.totalHits);
if (topDocs.totalHits == 1) {
- assertEquals(
- MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc),
- MultiDocValues.getNumericValues(reader2, "foo").get(topDocs2.scoreDocs[0].doc));
+ NumericDocValues dvs1 = MultiDocValues.getNumericValues(reader, "foo");
+ int hitDoc1 = topDocs.scoreDocs[0].doc;
+ assertEquals(hitDoc1, dvs1.advance(hitDoc1));
+ long value1 = dvs1.longValue();
+ NumericDocValues dvs2 = MultiDocValues.getNumericValues(reader2, "foo");
+ int hitDoc2 = topDocs2.scoreDocs[0].doc;
+ assertEquals(hitDoc2, dvs2.advance(hitDoc2));
+ long value2 = dvs2.longValue();
+ assertEquals(value1, value2);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 7a47d97..6f112a7 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -2555,7 +2555,9 @@ public class TestIndexWriter extends LuceneTestCase {
w.commit();
w.close();
DirectoryReader r = DirectoryReader.open(d);
- assertEquals(0, getOnlyLeafReader(r).getNormValues("foo").get(0));
+ NumericDocValues norms = getOnlyLeafReader(r).getNormValues("foo");
+ assertEquals(0, norms.nextDoc());
+ assertEquals(0, norms.longValue());
r.close();
d.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 4992338..6fdfca8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -2039,8 +2039,12 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
BinaryDocValues bcf = reader.getBinaryDocValues("bcf");
for (int i = 0; i < reader.maxDoc(); i++) {
if (liveDocs == null || liveDocs.get(i)) {
- assertEquals("doc=" + (docBase + i), cf.get(i), f.get(i) * 2);
- assertEquals("doc=" + (docBase + i), TestBinaryDocValuesUpdates.getValue(bcf, i), TestBinaryDocValuesUpdates.getValue(bf, i) * 2);
+ assertEquals(i, f.advance(i));
+ assertEquals(i, cf.advance(i));
+ assertEquals(i, bf.advance(i));
+ assertEquals(i, bcf.advance(i));
+ assertEquals("doc=" + (docBase + i), cf.longValue(), f.longValue() * 2);
+ assertEquals("doc=" + (docBase + i), TestBinaryDocValuesUpdates.getValue(bcf), TestBinaryDocValuesUpdates.getValue(bf) * 2);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java
index 55aa6e0..52f806a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java
@@ -433,16 +433,18 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(indexCommits.get(i));
IndexSearcher s = new IndexSearcher(r);
- NumericDocValues docValues = MultiDocValues.getNumericValues(r, "thread");
for(int id=0;id<idCount;id++) {
//System.out.println("TEST: check id=" + id + " expectedThreadID=" + expectedThreadIDs[id]);
TopDocs hits = s.search(new TermQuery(new Term("id", ""+id)), 1);
+ NumericDocValues docValues = MultiDocValues.getNumericValues(r, "thread");
// We pre-add all ids up front:
assert expectedThreadIDs[id] != -1;
assertEquals(1, hits.totalHits);
- int actualThreadID = (int) docValues.get(hits.scoreDocs[0].doc);
+ int hitDoc = hits.scoreDocs[0].doc;
+ assertEquals(hitDoc, docValues.advance(hitDoc));
+ int actualThreadID = (int) docValues.longValue();
if (expectedThreadIDs[id] != actualThreadID) {
System.out.println("FAIL: commit=" + i + " (of " + commits.size() + ") id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs actualThreadID=" + actualThreadID + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads + " reader=" + r + " commit=" + indexCommits.get(i));
for(int threadID=0;threadID<threadOps.size();threadID++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
index 29a7cd1..900ec2b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
@@ -70,7 +70,8 @@ public class TestMaxTermFrequency extends LuceneTestCase {
public void test() throws Exception {
NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(expected.get(i).intValue(), fooNorms.get(i) & 0xff);
+ assertEquals(i, fooNorms.nextDoc());
+ assertEquals(expected.get(i).intValue(), fooNorms.longValue() & 0xff);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
index 1f43ddd..f40379f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
@@ -122,7 +122,6 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
String f = "f" + field;
BinaryDocValues bdv = r.getBinaryDocValues(f);
NumericDocValues ndv = r.getNumericDocValues(f);
- Bits docsWithField = r.getDocsWithField(f);
if (field < numNDVFields) {
assertNotNull(ndv);
assertNull(bdv);
@@ -134,11 +133,12 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
- assertTrue(docsWithField.get(doc));
if (field < numNDVFields) {
- assertEquals("invalid numeric value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
+ assertEquals(doc, ndv.advance(doc));
+ assertEquals("invalid numeric value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.longValue());
} else {
- assertEquals("invalid binary value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc));
+ assertEquals(doc, bdv.advance(doc));
+ assertEquals("invalid binary value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv));
}
}
}
@@ -264,15 +264,13 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
for (int i = 0; i < numFields; i++) {
BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
NumericDocValues control = r.getNumericDocValues("cf" + i);
- Bits docsWithBdv = r.getDocsWithField("f" + i);
- Bits docsWithControl = r.getDocsWithField("cf" + i);
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
- assertTrue(docsWithBdv.get(j));
- assertTrue(docsWithControl.get(j));
- long ctrlValue = control.get(j);
- long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv, j) * 2;
+ assertEquals(j, control.advance(j));
+ long ctrlValue = control.longValue();
+ assertEquals(j, bdv.advance(j));
+ long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv) * 2;
// if (ctrlValue != bdvValue) {
// System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.document(j).get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
// }
@@ -315,7 +313,9 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
BinaryDocValues fbdv = r.getBinaryDocValues("f");
NumericDocValues cfndv = r.getNumericDocValues("cf");
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals(cfndv.get(j), TestBinaryDocValuesUpdates.getValue(fbdv, j) * 2);
+ assertEquals(j, cfndv.nextDoc());
+ assertEquals(j, fbdv.nextDoc());
+ assertEquals(cfndv.longValue(), TestBinaryDocValuesUpdates.getValue(fbdv) * 2);
}
}
reader.close();
@@ -384,7 +384,9 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
BinaryDocValues f = r.getBinaryDocValues("f" + i);
NumericDocValues cf = r.getNumericDocValues("cf" + i);
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.get(j), TestBinaryDocValuesUpdates.getValue(f, j) * 2);
+ assertEquals(j, cf.nextDoc());
+ assertEquals(j, f.nextDoc());
+ assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.longValue(), TestBinaryDocValuesUpdates.getValue(f) * 2);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
index 5b70c38..f6669d5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
@@ -17,6 +17,7 @@
package org.apache.lucene.index;
+import java.io.IOException;
import java.util.ArrayList;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -26,12 +27,15 @@ import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+
/** Tests MultiDocValues versus ordinary segment merging */
public class TestMultiDocValues extends LuceneTestCase {
@@ -62,8 +66,12 @@ public class TestMultiDocValues extends LuceneTestCase {
NumericDocValues multi = MultiDocValues.getNumericValues(ir, "numbers");
NumericDocValues single = merged.getNumericDocValues("numbers");
for (int i = 0; i < numDocs; i++) {
- assertEquals(single.get(i), multi.get(i));
+ assertEquals(i, multi.nextDoc());
+ assertEquals(i, single.nextDoc());
+ assertEquals(single.longValue(), multi.longValue());
}
+ testRandomAdvance(merged.getNumericDocValues("numbers"), MultiDocValues.getNumericValues(ir, "numbers"));
+
ir.close();
ir2.close();
dir.close();
@@ -80,6 +88,7 @@ public class TestMultiDocValues extends LuceneTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
+
for (int i = 0; i < numDocs; i++) {
BytesRef ref = new BytesRef(TestUtil.randomUnicodeString(random()));
field.setBytesValue(ref);
@@ -93,14 +102,18 @@ public class TestMultiDocValues extends LuceneTestCase {
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
-
+
BinaryDocValues multi = MultiDocValues.getBinaryValues(ir, "bytes");
BinaryDocValues single = merged.getBinaryDocValues("bytes");
for (int i = 0; i < numDocs; i++) {
- final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
- final BytesRef actual = multi.get(i);
+ assertEquals(i, multi.nextDoc());
+ assertEquals(i, single.nextDoc());
+ final BytesRef expected = BytesRef.deepCopyOf(single.binaryValue());
+ final BytesRef actual = multi.binaryValue();
assertEquals(expected, actual);
}
+ testRandomAdvance(merged.getBinaryDocValues("bytes"), MultiDocValues.getBinaryValues(ir, "bytes"));
+
ir.close();
ir2.close();
dir.close();
@@ -133,18 +146,24 @@ public class TestMultiDocValues extends LuceneTestCase {
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
-
SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes");
SortedDocValues single = merged.getSortedDocValues("bytes");
assertEquals(single.getValueCount(), multi.getValueCount());
- for (int i = 0; i < numDocs; i++) {
- // check ord
- assertEquals(single.getOrd(i), multi.getOrd(i));
+ while (true) {
+ assertEquals(single.nextDoc(), multi.nextDoc());
+ if (single.docID() == NO_MORE_DOCS) {
+ break;
+ }
+
// check value
- final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
- final BytesRef actual = multi.get(i);
+ final BytesRef expected = BytesRef.deepCopyOf(single.binaryValue());
+ final BytesRef actual = multi.binaryValue();
assertEquals(expected, actual);
+
+ // check ord
+ assertEquals(single.ordValue(), multi.ordValue());
}
+ testRandomAdvance(merged.getSortedDocValues("bytes"), MultiDocValues.getSortedValues(ir, "bytes"));
ir.close();
ir2.close();
dir.close();
@@ -180,13 +199,17 @@ public class TestMultiDocValues extends LuceneTestCase {
SortedDocValues single = merged.getSortedDocValues("bytes");
assertEquals(single.getValueCount(), multi.getValueCount());
for (int i = 0; i < numDocs; i++) {
+ assertEquals(i, multi.nextDoc());
+ assertEquals(i, single.nextDoc());
// check ord
- assertEquals(single.getOrd(i), multi.getOrd(i));
+ assertEquals(single.ordValue(), multi.ordValue());
// check ord value
- final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
- final BytesRef actual = multi.get(i);
+ final BytesRef expected = BytesRef.deepCopyOf(single.binaryValue());
+ final BytesRef actual = multi.binaryValue();
assertEquals(expected, actual);
}
+ testRandomAdvance(merged.getSortedDocValues("bytes"), MultiDocValues.getSortedValues(ir, "bytes"));
+
ir.close();
ir2.close();
dir.close();
@@ -230,15 +253,19 @@ public class TestMultiDocValues extends LuceneTestCase {
assertEquals(expected, actual);
}
// check ord list
- for (int i = 0; i < numDocs; i++) {
- single.setDocument(i);
+ while (true) {
+ int docID = single.nextDoc();
+ assertEquals(docID, multi.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+
ArrayList<Long> expectedList = new ArrayList<>();
long ord;
while ((ord = single.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
expectedList.add(ord);
}
- multi.setDocument(i);
int upto = 0;
while ((ord = multi.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
assertEquals(expectedList.get(upto).longValue(), ord);
@@ -247,6 +274,7 @@ public class TestMultiDocValues extends LuceneTestCase {
assertEquals(expectedList.size(), upto);
}
}
+ testRandomAdvance(merged.getSortedSetDocValues("bytes"), MultiDocValues.getSortedSetValues(ir, "bytes"));
ir.close();
ir2.close();
@@ -292,15 +320,18 @@ public class TestMultiDocValues extends LuceneTestCase {
assertEquals(expected, actual);
}
// check ord list
- for (int i = 0; i < numDocs; i++) {
- single.setDocument(i);
+ while (true) {
+ int docID = single.nextDoc();
+ assertEquals(docID, multi.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
ArrayList<Long> expectedList = new ArrayList<>();
long ord;
while ((ord = single.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
expectedList.add(ord);
}
- multi.setDocument(i);
int upto = 0;
while ((ord = multi.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
assertEquals(expectedList.get(upto).longValue(), ord);
@@ -309,6 +340,7 @@ public class TestMultiDocValues extends LuceneTestCase {
assertEquals(expectedList.size(), upto);
}
}
+ testRandomAdvance(merged.getSortedSetDocValues("bytes"), MultiDocValues.getSortedSetValues(ir, "bytes"));
ir.close();
ir2.close();
@@ -347,69 +379,35 @@ public class TestMultiDocValues extends LuceneTestCase {
} else {
// check values
for (int i = 0; i < numDocs; i++) {
- single.setDocument(i);
- ArrayList<Long> expectedList = new ArrayList<>();
- for (int j = 0; j < single.count(); j++) {
- expectedList.add(single.valueAt(j));
+ if (i > single.docID()) {
+ assertEquals(single.nextDoc(), multi.nextDoc());
}
-
- multi.setDocument(i);
- assertEquals(expectedList.size(), multi.count());
- for (int j = 0; j < single.count(); j++) {
- assertEquals(expectedList.get(j).longValue(), multi.valueAt(j));
+ if (i == single.docID()) {
+ assertEquals(single.docValueCount(), multi.docValueCount());
+ for (int j = 0; j < single.docValueCount(); j++) {
+ assertEquals(single.nextValue(), multi.nextValue());
+ }
}
}
}
+ testRandomAdvance(merged.getSortedNumericDocValues("nums"), MultiDocValues.getSortedNumericValues(ir, "nums"));
ir.close();
ir2.close();
dir.close();
}
-
- public void testDocsWithField() throws Exception {
- Directory dir = newDirectory();
-
- IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
- int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- if (random().nextInt(4) >= 0) {
- doc.add(new NumericDocValuesField("numbers", random().nextLong()));
- }
- doc.add(new NumericDocValuesField("numbersAlways", random().nextLong()));
- iw.addDocument(doc);
- if (random().nextInt(17) == 0) {
- iw.commit();
- }
- }
- DirectoryReader ir = iw.getReader();
- iw.forceMerge(1);
- DirectoryReader ir2 = iw.getReader();
- LeafReader merged = getOnlyLeafReader(ir2);
- iw.close();
-
- Bits multi = MultiDocValues.getDocsWithField(ir, "numbers");
- Bits single = merged.getDocsWithField("numbers");
- if (multi == null) {
- assertNull(single);
- } else {
- assertEquals(single.length(), multi.length());
- for (int i = 0; i < numDocs; i++) {
- assertEquals(single.get(i), multi.get(i));
+ private void testRandomAdvance(DocIdSetIterator iter1, DocIdSetIterator iter2) throws IOException {
+ assertEquals(-1, iter1.docID());
+ assertEquals(-1, iter2.docID());
+
+ while (iter1.docID() != NO_MORE_DOCS) {
+ if (random().nextBoolean()) {
+ assertEquals(iter1.nextDoc(), iter2.nextDoc());
+ } else {
+ int target = iter1.docID() + TestUtil.nextInt(random(), 1, 100);
+ assertEquals(iter1.advance(target), iter2.advance(target));
}
}
-
- multi = MultiDocValues.getDocsWithField(ir, "numbersAlways");
- single = merged.getDocsWithField("numbersAlways");
- assertEquals(single.length(), multi.length());
- for (int i = 0; i < numDocs; i++) {
- assertEquals(single.get(i), multi.get(i));
- }
- ir.close();
- ir2.close();
- dir.close();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
index ae35963..a51f1d9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
@@ -94,12 +94,14 @@ public class TestNorms extends LuceneTestCase {
NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(0, fooNorms.get(i));
+ assertEquals(i, fooNorms.nextDoc());
+ assertEquals(0, fooNorms.longValue());
}
NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(1, barNorms.get(i));
+ assertEquals(i, barNorms.nextDoc());
+ assertEquals(1, barNorms.longValue());
}
reader.close();
@@ -115,7 +117,8 @@ public class TestNorms extends LuceneTestCase {
for (int i = 0; i < open.maxDoc(); i++) {
Document document = open.document(i);
int expected = Integer.parseInt(document.get(byteTestField));
- assertEquals(expected, normValues.get(i));
+ assertEquals(i, normValues.nextDoc());
+ assertEquals(expected, normValues.longValue());
}
open.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
index 219dd05..94da587 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
@@ -221,8 +221,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
assertEquals(1, reader.leaves().size());
LeafReader r = reader.leaves().get(0).reader();
NumericDocValues ndv = r.getNumericDocValues("val");
- assertEquals(2, ndv.get(0));
- assertEquals(2, ndv.get(1));
+ assertEquals(0, ndv.nextDoc());
+ assertEquals(2, ndv.longValue());
+ assertEquals(1, ndv.nextDoc());
+ assertEquals(2, ndv.longValue());
reader.close();
dir.close();
@@ -267,7 +269,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
assertNotNull(ndv);
for (int i = 0; i < r.maxDoc(); i++) {
long expected = expectedValues[i + context.docBase];
- long actual = ndv.get(i);
+ assertEquals(i, ndv.nextDoc());
+ long actual = ndv.longValue();
assertEquals(expected, actual);
}
}
@@ -303,9 +306,13 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader2 = DirectoryReader.openIfChanged(reader1);
assertNotNull(reader2);
assertTrue(reader1 != reader2);
+ NumericDocValues dvs1 = reader1.leaves().get(0).reader().getNumericDocValues("val");
+ assertEquals(0, dvs1.nextDoc());
+ assertEquals(1, dvs1.longValue());
- assertEquals(1, reader1.leaves().get(0).reader().getNumericDocValues("val").get(0));
- assertEquals(10, reader2.leaves().get(0).reader().getNumericDocValues("val").get(0));
+ NumericDocValues dvs2 = reader2.leaves().get(0).reader().getNumericDocValues("val");
+ assertEquals(0, dvs2.nextDoc());
+ assertEquals(10, dvs2.longValue());
writer.close();
IOUtils.close(reader1, reader2, dir);
@@ -353,7 +360,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17};
NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "val");
for (int i = 0; i < expectedValues.length; i++) {
- assertEquals(expectedValues[i], ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(expectedValues[i], ndv.longValue());
}
reader.close();
@@ -389,7 +397,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
LeafReader r = reader.leaves().get(0).reader();
assertFalse(r.getLiveDocs().get(0));
- assertEquals(17, r.getNumericDocValues("val").get(1));
+ NumericDocValues values = r.getNumericDocValues("val");
+ assertEquals(1, values.advance(1));
+ assertEquals(17, values.longValue());
reader.close();
dir.close();
@@ -425,12 +435,16 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
SortedDocValues sdv = r.getSortedDocValues("sdv");
SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(17, ndv.get(i));
- BytesRef term = bdv.get(i);
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(17, ndv.longValue());
+ assertEquals(i, bdv.nextDoc());
+ BytesRef term = bdv.binaryValue();
assertEquals(new BytesRef(Integer.toString(i)), term);
- term = sdv.get(i);
+ assertEquals(i, sdv.nextDoc());
+ term = sdv.binaryValue();
assertEquals(new BytesRef(Integer.toString(i)), term);
- ssdv.setDocument(i);
+ assertEquals(i, ssdv.nextDoc());
+
long ord = ssdv.nextOrd();
term = ssdv.lookupOrd(ord);
assertEquals(i, Integer.parseInt(term.utf8ToString()));
@@ -471,8 +485,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues ndv1 = r.getNumericDocValues("ndv1");
NumericDocValues ndv2 = r.getNumericDocValues("ndv2");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(17, ndv1.get(i));
- assertEquals(i, ndv2.get(i));
+ assertEquals(i, ndv1.nextDoc());
+ assertEquals(17, ndv1.longValue());
+ assertEquals(i, ndv2.nextDoc());
+ assertEquals(i, ndv2.longValue());
}
reader.close();
@@ -503,7 +519,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
LeafReader r = reader.leaves().get(0).reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(17, ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(17, ndv.longValue());
}
reader.close();
@@ -567,8 +584,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv");
SortedDocValues sdv = MultiDocValues.getSortedValues(reader, "sorted");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(17, ndv.get(i));
- final BytesRef term = sdv.get(i);
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(17, ndv.longValue());
+ assertEquals(i, sdv.nextDoc());
+ final BytesRef term = sdv.binaryValue();
assertEquals(new BytesRef("value"), term);
}
@@ -596,7 +615,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(3, ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(3, ndv.longValue());
}
reader.close();
dir.close();
@@ -665,7 +685,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues ndv = r.getNumericDocValues("ndv");
assertNotNull(ndv);
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(value, ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(value, ndv.longValue());
}
reader.close();
}
@@ -696,7 +717,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(3, ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(3, ndv.longValue());
}
reader.close();
dir.close();
@@ -774,14 +796,13 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
for (int field = 0; field < fieldValues.length; field++) {
String f = "f" + field;
NumericDocValues ndv = r.getNumericDocValues(f);
- Bits docsWithField = r.getDocsWithField(f);
assertNotNull(ndv);
int maxDoc = r.maxDoc();
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + ndv.get(doc));
- assertTrue(docsWithField.get(doc));
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
+ assertEquals(doc, ndv.advance(doc));
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.longValue());
}
}
}
@@ -835,12 +856,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
for (LeafReaderContext context : reader.leaves()) {
LeafReader r = context.reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
- Bits docsWithField = r.getDocsWithField("ndv");
- assertNotNull(docsWithField);
- assertTrue(docsWithField.get(0));
- assertEquals(5L, ndv.get(0));
- assertFalse(docsWithField.get(1));
- assertEquals(0L, ndv.get(1));
+ assertEquals(0, ndv.nextDoc());
+ assertEquals(5L, ndv.longValue());
+ // docID 1 has no ndv value
+ assertTrue(ndv.nextDoc() > 1);
}
reader.close();
@@ -890,12 +909,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
for (LeafReaderContext context : reader.leaves()) {
LeafReader r = context.reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
- Bits docsWithField = r.getDocsWithField("ndv");
- assertNotNull(docsWithField);
- assertTrue(docsWithField.get(0));
- assertEquals(5L, ndv.get(0));
- assertFalse(docsWithField.get(1));
- assertEquals(0L, ndv.get(1));
+ assertEquals(0, ndv.nextDoc());
+ assertEquals(5L, ndv.longValue());
+ assertTrue(ndv.nextDoc() > 1);
}
reader.close();
@@ -967,7 +983,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
LeafReader r = context.reader();
NumericDocValues ndv = r.getNumericDocValues("ndv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(5L, ndv.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(5L, ndv.longValue());
}
}
reader.close();
@@ -993,7 +1010,8 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(dir);
NumericDocValues ndv = r.leaves().get(0).reader().getNumericDocValues("f");
- assertEquals(17, ndv.get(0));
+ assertEquals(0, ndv.nextDoc());
+ assertEquals(17, ndv.longValue());
r.close();
dir.close();
@@ -1111,14 +1129,12 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
for (int i = 0; i < numFields; i++) {
NumericDocValues ndv = r.getNumericDocValues("f" + i);
NumericDocValues control = r.getNumericDocValues("cf" + i);
- Bits docsWithNdv = r.getDocsWithField("f" + i);
- Bits docsWithControl = r.getDocsWithField("cf" + i);
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
- assertTrue(docsWithNdv.get(j));
- assertTrue(docsWithControl.get(j));
- assertEquals(control.get(j), ndv.get(j) * 2);
+ assertEquals(j, ndv.advance(j));
+ assertEquals(j, control.advance(j));
+ assertEquals(control.longValue(), ndv.longValue() * 2);
}
}
}
@@ -1157,7 +1173,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues fndv = r.getNumericDocValues("f");
NumericDocValues cfndv = r.getNumericDocValues("cf");
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals(cfndv.get(j), fndv.get(j) * 2);
+ assertEquals(j, fndv.nextDoc());
+ assertEquals(j, cfndv.nextDoc());
+ assertEquals(cfndv.longValue(), fndv.longValue() * 2);
}
}
reader.close();
@@ -1206,10 +1224,14 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
NumericDocValues f1 = MultiDocValues.getNumericValues(reader, "f1");
NumericDocValues f2 = MultiDocValues.getNumericValues(reader, "f2");
- assertEquals(12L, f1.get(0));
- assertEquals(13L, f2.get(0));
- assertEquals(17L, f1.get(1));
- assertEquals(2L, f2.get(1));
+ assertEquals(0, f1.nextDoc());
+ assertEquals(12L, f1.longValue());
+ assertEquals(0, f2.nextDoc());
+ assertEquals(13L, f2.longValue());
+ assertEquals(1, f1.nextDoc());
+ assertEquals(17L, f1.longValue());
+ assertEquals(1, f2.nextDoc());
+ assertEquals(2L, f2.longValue());
reader.close();
dir.close();
}
@@ -1264,7 +1286,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues ndv = r.getNumericDocValues("ndv");
NumericDocValues control = r.getNumericDocValues("control");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(ndv.get(i)*2, control.get(i));
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(i, control.nextDoc());
+ assertEquals(ndv.longValue()*2, control.longValue());
}
}
reader.close();
@@ -1361,7 +1385,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
NumericDocValues f = r.getNumericDocValues("f" + i);
NumericDocValues cf = r.getNumericDocValues("cf" + i);
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.get(j), f.get(j) * 2);
+ assertEquals(j, f.nextDoc());
+ assertEquals(j, cf.nextDoc());
+ assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.longValue(), f.longValue() * 2);
}
}
}
@@ -1390,8 +1416,12 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
writer.close();
DirectoryReader reader = DirectoryReader.open(dir);
- assertEquals(4, reader.leaves().get(0).reader().getNumericDocValues("f1").get(0));
- assertEquals(3, reader.leaves().get(0).reader().getNumericDocValues("f2").get(0));
+ NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(4, dvs.longValue());
+ dvs = reader.leaves().get(0).reader().getNumericDocValues("f2");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(3, dvs.longValue());
reader.close();
dir.close();
@@ -1416,7 +1446,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
- assertEquals(2L, reader.leaves().get(0).reader().getNumericDocValues("f1").get(0));
+ NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(2, dvs.longValue());
+
reader.close();
dir.close();
@@ -1439,7 +1472,9 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
- assertEquals(1L, reader.leaves().get(0).reader().getNumericDocValues("f1").get(0));
+ NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(1, dvs.longValue());
reader.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
index bc3c3e8..bee1720 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
@@ -29,6 +29,8 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestOmitNorms extends LuceneTestCase {
// Tests whether the DocumentWriter correctly enable the
// omitNorms bit in the FieldInfo
@@ -302,8 +304,21 @@ public class TestOmitNorms extends LuceneTestCase {
if (norms1 == null) {
assertNull(norms2);
} else {
- for(int docID=0;docID<ir1.maxDoc();docID++) {
- assertEquals(norms1.get(docID), norms2.get(docID));
+ while (true) {
+ int norms1DocID = norms1.nextDoc();
+ int norms2DocID = norms2.nextDoc();
+ while (norms1DocID < norms2DocID) {
+ assertEquals(0, norms1.longValue());
+ norms1DocID = norms1.nextDoc();
+ }
+ while (norms2DocID < norms1DocID) {
+ assertEquals(0, norms2.longValue());
+ norms2DocID = norms2.nextDoc();
+ }
+ if (norms1.docID() == NO_MORE_DOCS) {
+ break;
+ }
+ assertEquals(norms1.longValue(), norms2.longValue());
}
}
ir1.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java b/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
index 1f9ff11..921102d6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
@@ -25,8 +25,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
-import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -84,13 +82,13 @@ public class TestOrdinalMap extends LuceneTestCase {
iw.commit();
DirectoryReader r = iw.getReader();
SortedDocValues sdv = MultiDocValues.getSortedValues(r, "sdv");
- if (sdv instanceof MultiSortedDocValues) {
- OrdinalMap map = ((MultiSortedDocValues) sdv).mapping;
+ if (sdv instanceof MultiDocValues.MultiSortedDocValues) {
+ OrdinalMap map = ((MultiDocValues.MultiSortedDocValues) sdv).mapping;
assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
SortedSetDocValues ssdv = MultiDocValues.getSortedSetValues(r, "ssdv");
- if (ssdv instanceof MultiSortedSetDocValues) {
- OrdinalMap map = ((MultiSortedSetDocValues) ssdv).mapping;
+ if (ssdv instanceof MultiDocValues.MultiSortedSetDocValues) {
+ OrdinalMap map = ((MultiDocValues.MultiSortedSetDocValues) ssdv).mapping;
assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
iw.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
index b21cb23..c265d92 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
@@ -295,7 +295,13 @@ public class TestPostingsOffsets extends LuceneTestCase {
PostingsEnum docs = null;
PostingsEnum docsAndPositions = null;
PostingsEnum docsAndPositionsAndOffsets = null;
- final NumericDocValues docIDToID = DocValues.getNumeric(sub, "id");
+ int[] docIDToID = new int[sub.maxDoc()];
+ NumericDocValues values = DocValues.getNumeric(sub, "id");
+ for(int i=0;i<sub.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToID[i] = (int) values.longValue();
+ }
+
for(String term : terms) {
//System.out.println(" term=" + term);
if (termsEnum.seekExact(new BytesRef(term))) {
@@ -304,8 +310,8 @@ public class TestPostingsOffsets extends LuceneTestCase {
int doc;
//System.out.println(" doc/freq");
while((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- final List<Token> expected = actualTokens.get(term).get((int) docIDToID.get(doc));
- //System.out.println(" doc=" + docIDToID.get(doc) + " docID=" + doc + " " + expected.size() + " freq");
+ final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+ //System.out.println(" doc=" + docIDToID[doc] + " docID=" + doc + " " + expected.size() + " freq");
assertNotNull(expected);
assertEquals(expected.size(), docs.freq());
}
@@ -315,8 +321,8 @@ public class TestPostingsOffsets extends LuceneTestCase {
assertNotNull(docsAndPositions);
//System.out.println(" doc/freq/pos");
while((doc = docsAndPositions.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- final List<Token> expected = actualTokens.get(term).get((int) docIDToID.get(doc));
- //System.out.println(" doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
+ final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+ //System.out.println(" doc=" + docIDToID[doc] + " " + expected.size() + " freq");
assertNotNull(expected);
assertEquals(expected.size(), docsAndPositions.freq());
for(Token token : expected) {
@@ -330,8 +336,8 @@ public class TestPostingsOffsets extends LuceneTestCase {
assertNotNull(docsAndPositionsAndOffsets);
//System.out.println(" doc/freq/pos/offs");
while((doc = docsAndPositionsAndOffsets.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- final List<Token> expected = actualTokens.get(term).get((int) docIDToID.get(doc));
- //System.out.println(" doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
+ final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+ //System.out.println(" doc=" + docIDToID[doc] + " " + expected.size() + " freq");
assertNotNull(expected);
assertEquals(expected.size(), docsAndPositionsAndOffsets.freq());
for(Token token : expected) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
index b074f81..6015fbc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
@@ -224,7 +224,12 @@ public class TestTermsEnum extends LuceneTestCase {
final IndexReader r = w.getReader();
w.close();
- final NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+ int[] docIDToID = new int[r.maxDoc()];
+ NumericDocValues values = MultiDocValues.getNumericValues(r, "id");
+ for(int i=0;i<r.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToID[i] = (int) values.longValue();
+ }
for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
@@ -327,7 +332,7 @@ public class TestTermsEnum extends LuceneTestCase {
postingsEnum = TestUtil.docs(random(), te, postingsEnum, PostingsEnum.NONE);
final int docID = postingsEnum.nextDoc();
assertTrue(docID != DocIdSetIterator.NO_MORE_DOCS);
- assertEquals(docIDToID.get(docID), termToID.get(expected).intValue());
+ assertEquals(docIDToID[docID], termToID.get(expected).intValue());
do {
loc++;
} while (loc < termsArray.length && !acceptTermsSet.contains(termsArray[loc]));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
index b1661ca..ee47fff 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
@@ -72,7 +72,8 @@ public class TestUniqueTermCount extends LuceneTestCase {
NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
assertNotNull(fooNorms);
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(expected.get(i).longValue(), fooNorms.get(i));
+ assertEquals(i, fooNorms.nextDoc());
+ assertEquals(expected.get(i).longValue(), fooNorms.longValue());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
index de40c0d..ac35631 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
@@ -22,10 +22,10 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -153,9 +153,25 @@ public class TestDocValuesScoring extends LuceneTestCase {
final NumericDocValues values = DocValues.getNumeric(context.reader(), boostField);
return new SimScorer() {
+
+ private float getValueForDoc(int doc) throws IOException {
+ int curDocID = values.docID();
+ if (doc < curDocID) {
+ throw new IllegalArgumentException("doc=" + doc + " is before curDocID=" + curDocID);
+ }
+ if (doc > curDocID) {
+ curDocID = values.advance(doc);
+ }
+ if (curDocID == doc) {
+ return Float.intBitsToFloat((int)values.longValue());
+ } else {
+ return 0f;
+ }
+ }
+
@Override
- public float score(int doc, float freq) {
- return Float.intBitsToFloat((int)values.get(doc)) * sub.score(doc, freq);
+ public float score(int doc, float freq) throws IOException {
+ return getValueForDoc(doc) * sub.score(doc, freq);
}
@Override
@@ -169,8 +185,8 @@ public class TestDocValuesScoring extends LuceneTestCase {
}
@Override
- public Explanation explain(int doc, Explanation freq) {
- Explanation boostExplanation = Explanation.match(Float.intBitsToFloat((int)values.get(doc)), "indexDocValue(" + boostField + ")");
+ public Explanation explain(int doc, Explanation freq) throws IOException {
+ Explanation boostExplanation = Explanation.match(getValueForDoc(doc), "indexDocValue(" + boostField + ")");
Explanation simExplanation = sub.explain(doc, freq);
return Explanation.match(
boostExplanation.getValue() * simExplanation.getValue(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java b/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
index 7fdc367..9ca2302 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
@@ -151,9 +151,7 @@ class ElevationComparatorSource extends FieldComparatorSource {
int bottomVal;
@Override
- public LeafFieldComparator getLeafComparator(LeafReaderContext context)
- throws IOException {
- final SortedDocValues idIndex = DocValues.getSorted(context.reader(), fieldname);
+ public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
return new LeafFieldComparator() {
@Override
@@ -166,24 +164,24 @@ class ElevationComparatorSource extends FieldComparatorSource {
throw new UnsupportedOperationException();
}
- private int docVal(int doc) {
- int ord = idIndex.getOrd(doc);
- if (ord == -1) {
- return 0;
- } else {
- final BytesRef term = idIndex.lookupOrd(ord);
+ private int docVal(int doc) throws IOException {
+ SortedDocValues idIndex = DocValues.getSorted(context.reader(), fieldname);
+ if (idIndex.advance(doc) == doc) {
+ final BytesRef term = idIndex.binaryValue();
Integer prio = priority.get(term);
return prio == null ? 0 : prio.intValue();
+ } else {
+ return 0;
}
}
@Override
- public int compareBottom(int doc) {
+ public int compareBottom(int doc) throws IOException {
return docVal(doc) - bottomVal;
}
@Override
- public void copy(int slot, int doc) {
+ public void copy(int slot, int doc) throws IOException {
values[slot] = docVal(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
index 19f0be5..b2c21b3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
@@ -367,7 +367,12 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
for (currentDoc = currentDoc+1; currentDoc < maxDoc; currentDoc++) {
currentMatched = 0;
score = 0;
- dv.setDocument(currentDoc);
+ if (currentDoc > dv.docID()) {
+ dv.advance(currentDoc);
+ }
+ if (currentDoc != dv.docID()) {
+ continue;
+ }
long ord;
while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
if (ords.contains(ord)) {
[16/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
LUCENE-7407: switch doc values usage to an iterator API, based on DocIdSetIterator, instead of random acces, freeing codecs for future improvements
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f7aa200d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f7aa200d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f7aa200d
Branch: refs/heads/master
Commit: f7aa200d406dbd05a35d6116198302d90b92cb29
Parents: 93bc88d
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Sep 21 09:41:41 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Sep 21 09:41:41 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../collation/TestCollationDocValuesField.java | 3 +-
.../TestICUCollationDocValuesField.java | 3 +-
.../index/TestBackwardsCompatibility.java | 61 +-
.../codecs/memory/DirectDocValuesConsumer.java | 33 +-
.../codecs/memory/DirectDocValuesProducer.java | 100 +-
.../codecs/memory/MemoryDocValuesConsumer.java | 45 +-
.../codecs/memory/MemoryDocValuesProducer.java | 104 +-
.../simpletext/SimpleTextDocValuesReader.java | 64 +-
.../simpletext/SimpleTextDocValuesWriter.java | 31 +-
.../simpletext/SimpleTextNormsFormat.java | 9 +-
.../apache/lucene/codecs/DocValuesConsumer.java | 1069 ++++++++----------
.../apache/lucene/codecs/DocValuesProducer.java | 7 -
.../lucene/codecs/LegacyDocValuesIterables.java | 514 +++++++++
.../org/apache/lucene/codecs/NormsConsumer.java | 133 +--
.../codecs/lucene53/Lucene53NormsConsumer.java | 8 +-
.../codecs/lucene53/Lucene53NormsProducer.java | 47 +-
.../lucene54/Lucene54DocValuesConsumer.java | 41 +-
.../lucene54/Lucene54DocValuesProducer.java | 345 ++++--
.../lucene/codecs/lucene54/package-info.java | 2 +-
.../perfield/PerFieldDocValuesFormat.java | 28 +-
.../lucene/document/BinaryDocValuesField.java | 1 -
.../apache/lucene/index/BinaryDocValues.java | 19 +-
.../lucene/index/BinaryDocValuesWriter.java | 80 +-
.../org/apache/lucene/index/CheckIndex.java | 178 ++-
.../org/apache/lucene/index/CodecReader.java | 151 +--
.../org/apache/lucene/index/DocIDMerger.java | 11 +-
.../java/org/apache/lucene/index/DocValues.java | 327 ++++--
.../lucene/index/EmptyDocValuesProducer.java | 71 ++
.../lucene/index/FilterBinaryDocValues.java | 61 +
.../apache/lucene/index/FilterLeafReader.java | 6 -
.../lucene/index/FilterNumericDocValues.java | 59 +
.../org/apache/lucene/index/LeafReader.java | 13 +-
.../lucene/index/LegacyBinaryDocValues.java | 39 +
.../index/LegacyBinaryDocValuesWrapper.java | 82 ++
.../lucene/index/LegacyNumericDocValues.java | 38 +
.../index/LegacyNumericDocValuesWrapper.java | 92 ++
.../lucene/index/LegacySortedDocValues.java | 110 ++
.../index/LegacySortedDocValuesWrapper.java | 92 ++
.../index/LegacySortedNumericDocValues.java | 49 +
.../LegacySortedNumericDocValuesWrapper.java | 88 ++
.../lucene/index/LegacySortedSetDocValues.java | 109 ++
.../index/LegacySortedSetDocValuesWrapper.java | 102 ++
.../apache/lucene/index/MergeReaderWrapper.java | 15 -
.../org/apache/lucene/index/MultiDocValues.java | 642 ++++++++---
.../org/apache/lucene/index/MultiSorter.java | 339 +++---
.../apache/lucene/index/NormValuesWriter.java | 69 +-
.../apache/lucene/index/NumericDocValues.java | 9 +-
.../index/NumericDocValuesFieldUpdates.java | 3 +-
.../lucene/index/NumericDocValuesWriter.java | 65 +-
.../apache/lucene/index/ParallelLeafReader.java | 9 +-
.../apache/lucene/index/RandomAccessOrds.java | 53 -
.../apache/lucene/index/ReadersAndUpdates.java | 216 ++--
.../lucene/index/SegmentDocValuesProducer.java | 8 -
.../org/apache/lucene/index/SegmentReader.java | 12 +-
.../index/SingletonSortedNumericDocValues.java | 55 +-
.../index/SingletonSortedSetDocValues.java | 47 +-
.../lucene/index/SlowCodecReaderWrapper.java | 7 +-
.../apache/lucene/index/SortedDocValues.java | 30 +-
.../lucene/index/SortedDocValuesWriter.java | 112 +-
.../lucene/index/SortedNumericDocValues.java | 21 +-
.../index/SortedNumericDocValuesWriter.java | 103 +-
.../apache/lucene/index/SortedSetDocValues.java | 22 +-
.../lucene/index/SortedSetDocValuesWriter.java | 191 ++--
.../java/org/apache/lucene/index/Sorter.java | 204 +++-
.../apache/lucene/index/SortingLeafReader.java | 460 ++++++--
.../lucene/search/DocValuesRewriteMethod.java | 18 +-
.../apache/lucene/search/ExactPhraseScorer.java | 2 +-
.../apache/lucene/search/FieldComparator.java | 288 ++---
.../apache/lucene/search/FieldValueQuery.java | 45 +-
.../lucene/search/SloppyPhraseScorer.java | 2 +-
.../lucene/search/SortedNumericSelector.java | 101 +-
.../lucene/search/SortedNumericSortField.java | 2 +-
.../apache/lucene/search/SortedSetSelector.java | 249 +++-
.../lucene/search/SortedSetSortField.java | 5 +-
.../org/apache/lucene/search/package-info.java | 2 +-
.../search/similarities/BM25Similarity.java | 31 +-
.../search/similarities/MultiSimilarity.java | 4 +-
.../lucene/search/similarities/Similarity.java | 4 +-
.../search/similarities/SimilarityBase.java | 26 +-
.../search/similarities/TFIDFSimilarity.java | 35 +-
.../src/java/org/apache/lucene/util/Bits.java | 4 +-
.../java/org/apache/lucene/util/LongValues.java | 11 +-
.../apache/lucene/util/packed/PackedInts.java | 4 +-
.../lucene54/TestLucene54DocValuesFormat.java | 82 +-
.../perfield/TestPerFieldDocValuesFormat.java | 10 +-
.../lucene/index/Test2BBinaryDocValues.java | 6 +-
.../lucene/index/Test2BNumericDocValues.java | 3 +-
.../index/Test2BSortedDocValuesFixedSorted.java | 5 +-
.../lucene/index/Test2BSortedDocValuesOrds.java | 5 +-
.../index/TestBinaryDocValuesUpdates.java | 139 ++-
.../apache/lucene/index/TestCustomNorms.java | 3 +-
.../index/TestDemoParallelLeafReader.java | 58 +-
.../lucene/index/TestDirectoryReader.java | 11 +-
.../lucene/index/TestDirectoryReaderReopen.java | 23 +-
.../org/apache/lucene/index/TestDocValues.java | 11 +-
.../lucene/index/TestDocValuesIndexing.java | 28 +-
.../apache/lucene/index/TestIndexSorting.java | 171 +--
.../apache/lucene/index/TestIndexWriter.java | 4 +-
.../lucene/index/TestIndexWriterExceptions.java | 8 +-
.../index/TestIndexingSequenceNumbers.java | 6 +-
.../lucene/index/TestMaxTermFrequency.java | 3 +-
.../lucene/index/TestMixedDocValuesUpdates.java | 26 +-
.../apache/lucene/index/TestMultiDocValues.java | 142 ++-
.../test/org/apache/lucene/index/TestNorms.java | 9 +-
.../index/TestNumericDocValuesUpdates.java | 139 ++-
.../org/apache/lucene/index/TestOmitNorms.java | 19 +-
.../org/apache/lucene/index/TestOrdinalMap.java | 10 +-
.../lucene/index/TestPostingsOffsets.java | 20 +-
.../org/apache/lucene/index/TestTermsEnum.java | 9 +-
.../lucene/index/TestUniqueTermCount.java | 3 +-
.../lucene/search/TestDocValuesScoring.java | 26 +-
.../lucene/search/TestElevationComparator.java | 20 +-
.../lucene/search/TestMinShouldMatch2.java | 7 +-
.../apache/lucene/search/TestPointQueries.java | 13 +-
.../lucene/search/TestSimilarityProvider.java | 4 +-
.../apache/lucene/search/TestSortRandom.java | 3 +-
.../lucene/util/packed/TestDirectPacked.java | 6 +-
.../lucene/expressions/SimpleBindings.java | 2 +-
.../expressions/js/JavascriptCompiler.java | 2 +-
.../expressions/TestExpressionValueSource.java | 1 +
.../DefaultSortedSetDocValuesReaderState.java | 8 +-
.../SortedSetDocValuesFacetCounts.java | 54 +-
.../facet/taxonomy/DocValuesOrdinalsReader.java | 19 +-
.../facet/taxonomy/FastTaxonomyFacetCounts.java | 35 +-
.../taxonomy/OrdinalMappingLeafReader.java | 18 +-
.../facet/taxonomy/TaxonomyFacetCounts.java | 2 +-
.../TaxonomyFacetSumFloatAssociations.java | 39 +-
.../TaxonomyFacetSumIntAssociations.java | 39 +-
.../lucene/facet/TestMultipleIndexFields.java | 2 +-
.../taxonomy/TestOrdinalMappingLeafReader.java | 6 +-
.../AbstractFirstPassGroupingCollector.java | 2 +-
.../FunctionFirstPassGroupingCollector.java | 2 +-
.../term/TermAllGroupHeadsCollector.java | 170 ++-
.../grouping/term/TermAllGroupsCollector.java | 22 +-
.../term/TermDistinctValuesCollector.java | 37 +-
.../term/TermFirstPassGroupingCollector.java | 16 +-
.../grouping/term/TermGroupFacetCollector.java | 61 +-
.../term/TermSecondPassGroupingCollector.java | 17 +-
.../grouping/AllGroupHeadsCollectorTest.java | 11 +-
.../lucene/search/grouping/TestGrouping.java | 41 +-
.../search/highlight/TermVectorLeafReader.java | 5 -
.../highlight/WeightedSpanTermExtractor.java | 7 +-
.../search/join/BaseGlobalOrdinalScorer.java | 6 +-
.../lucene/search/join/BlockJoinSelector.java | 196 +++-
.../search/join/DocValuesTermsCollector.java | 5 +-
.../search/join/GenericTermsCollector.java | 30 +-
.../search/join/GlobalOrdinalsCollector.java | 21 +-
.../lucene/search/join/GlobalOrdinalsQuery.java | 21 +-
.../join/GlobalOrdinalsWithScoreCollector.java | 36 +-
.../join/GlobalOrdinalsWithScoreQuery.java | 30 +-
.../org/apache/lucene/search/join/JoinUtil.java | 44 +-
.../lucene/search/join/TermsCollector.java | 23 +-
.../search/join/TermsWithScoreCollector.java | 89 +-
.../search/join/ToParentBlockJoinSortField.java | 54 +-
.../search/join/TestBlockJoinSelector.java | 197 +++-
.../search/join/TestBlockJoinSorting.java | 9 +-
.../apache/lucene/search/join/TestJoinUtil.java | 51 +-
.../apache/lucene/index/memory/MemoryIndex.java | 88 +-
.../lucene/index/memory/TestMemoryIndex.java | 42 +-
.../memory/TestMemoryIndexAgainstRAMDir.java | 58 +-
.../search/DiversifiedTopDocsCollector.java | 14 +-
.../search/TestDiversifiedTopDocsCollector.java | 70 +-
.../lucene/queries/function/FunctionValues.java | 60 +-
.../lucene/queries/function/ValueSource.java | 10 +-
.../queries/function/ValueSourceScorer.java | 2 +-
.../function/docvalues/BoolDocValues.java | 24 +-
.../docvalues/DocTermsIndexDocValues.java | 78 +-
.../function/docvalues/DoubleDocValues.java | 32 +-
.../function/docvalues/FloatDocValues.java | 22 +-
.../function/docvalues/IntDocValues.java | 24 +-
.../function/docvalues/LongDocValues.java | 26 +-
.../function/docvalues/StrDocValues.java | 12 +-
.../valuesource/BytesRefFieldSource.java | 56 +-
.../valuesource/ComparisonBoolFunction.java | 8 +-
.../function/valuesource/DefFunction.java | 26 +-
.../function/valuesource/DivFloatFunction.java | 4 +-
.../function/valuesource/DoubleFieldSource.java | 43 +-
.../function/valuesource/DualFloatFunction.java | 8 +-
.../function/valuesource/EnumFieldSource.java | 37 +-
.../function/valuesource/FloatFieldSource.java | 40 +-
.../function/valuesource/IfFunction.java | 24 +-
.../function/valuesource/IntFieldSource.java | 40 +-
.../valuesource/JoinDocFreqValueSource.java | 27 +-
.../valuesource/LinearFloatFunction.java | 6 +-
.../function/valuesource/LongFieldSource.java | 56 +-
.../function/valuesource/MaxFloatFunction.java | 6 +-
.../function/valuesource/MinFloatFunction.java | 6 +-
.../function/valuesource/MultiBoolFunction.java | 6 +-
.../valuesource/MultiFloatFunction.java | 10 +-
.../function/valuesource/MultiFunction.java | 13 +-
.../function/valuesource/NormValueSource.java | 23 +-
.../function/valuesource/PowFloatFunction.java | 4 +-
.../valuesource/ProductFloatFunction.java | 4 +-
.../valuesource/RangeMapFloatFunction.java | 4 +-
.../valuesource/ReciprocalFloatFunction.java | 6 +-
.../valuesource/ScaleFloatFunction.java | 6 +-
.../valuesource/SimpleBoolFunction.java | 6 +-
.../valuesource/SimpleFloatFunction.java | 6 +-
.../valuesource/SortedSetFieldSource.java | 6 +-
.../function/valuesource/SumFloatFunction.java | 6 +-
.../function/valuesource/VectorValueSource.java | 33 +-
.../lucene/queries/TestCustomScoreQuery.java | 11 +-
.../queries/function/TestValueSources.java | 4 +-
.../document/LatLonPointDistanceComparator.java | 58 +-
.../lucene/search/DocValuesNumbersQuery.java | 31 +-
.../lucene/search/DocValuesRangeQuery.java | 36 +-
.../lucene/search/DocValuesTermsQuery.java | 16 +-
.../lucene/search/TermAutomatonScorer.java | 2 +-
.../search/BaseRangeFieldQueryTestCase.java | 5 +-
.../spatial/bbox/BBoxSimilarityValueSource.java | 9 +-
.../lucene/spatial/bbox/BBoxValueSource.java | 54 +-
.../serialized/SerializedDVStrategy.java | 35 +-
.../spatial/util/CachingDoubleValueSource.java | 6 +-
.../util/DistanceToShapeValueSource.java | 4 +-
.../spatial/util/ShapeAreaValueSource.java | 6 +-
.../spatial/util/ShapePredicateValueSource.java | 4 +-
.../spatial/vector/DistanceValueSource.java | 33 +-
.../GeoPointTermQueryConstantScoreWrapper.java | 16 +-
.../spatial3d/Geo3DPointDistanceComparator.java | 43 +-
.../Geo3DPointOutsideDistanceComparator.java | 41 +-
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 7 +-
.../search/suggest/DocumentDictionary.java | 12 +-
.../suggest/DocumentValueSourceDictionary.java | 8 +-
.../analyzing/AnalyzingInfixSuggester.java | 31 +-
.../analyzing/BlendedInfixSuggester.java | 26 +-
.../document/TestPrefixCompletionQuery.java | 33 +-
.../asserting/AssertingDocValuesFormat.java | 173 ++-
.../asserting/AssertingLiveDocsFormat.java | 2 +-
.../codecs/asserting/AssertingNormsFormat.java | 22 +-
.../codecs/cranky/CrankyDocValuesFormat.java | 21 +-
.../lucene/codecs/cranky/CrankyNormsFormat.java | 4 +-
.../apache/lucene/geo/BaseGeoPointTestCase.java | 21 +-
.../lucene/index/AssertingLeafReader.java | 356 +++---
.../index/BaseDocValuesFormatTestCase.java | 724 ++++++------
.../index/BaseIndexFileFormatTestCase.java | 101 +-
.../lucene/index/BaseNormsFormatTestCase.java | 14 +-
.../lucene/index/BasePointsFormatTestCase.java | 26 +-
.../index/BaseStoredFieldsFormatTestCase.java | 3 +-
.../lucene/index/FieldFilterLeafReader.java | 13 -
.../org/apache/lucene/search/QueryUtils.java | 11 +-
.../org/apache/lucene/util/LuceneTestCase.java | 77 +-
.../java/org/apache/lucene/util/TestUtil.java | 3 +-
.../facet/FieldFacetAccumulator.java | 44 +-
.../AbstractDelegatingStatsCollector.java | 2 +-
.../statistics/MedianStatsCollector.java | 3 +-
.../statistics/MinMaxStatsCollector.java | 2 +-
.../statistics/NumericStatsCollector.java | 3 +-
.../statistics/PercentileStatsCollector.java | 3 +-
.../analytics/statistics/StatsCollector.java | 2 +-
.../statistics/UniqueStatsCollector.java | 3 +-
.../AbsoluteValueDoubleFunction.java | 4 +-
.../util/valuesource/AddDoubleFunction.java | 4 +-
.../util/valuesource/ConcatStringFunction.java | 4 +-
.../util/valuesource/DateFieldSource.java | 36 +-
.../util/valuesource/DateMathFunction.java | 3 +-
.../util/valuesource/DivDoubleFunction.java | 4 +-
.../util/valuesource/DualDoubleFunction.java | 8 +-
.../util/valuesource/FilterFieldSource.java | 22 +-
.../util/valuesource/LogDoubleFunction.java | 4 +-
.../util/valuesource/MultiDateFunction.java | 10 +-
.../util/valuesource/MultiDoubleFunction.java | 8 +-
.../util/valuesource/MultiStringFunction.java | 12 +-
.../valuesource/MultiplyDoubleFunction.java | 4 +-
.../util/valuesource/NegateDoubleFunction.java | 4 +-
.../util/valuesource/PowDoubleFunction.java | 4 +-
.../util/valuesource/ReverseStringFunction.java | 4 +-
.../util/valuesource/SingleDoubleFunction.java | 8 +-
.../util/valuesource/SingleStringFunction.java | 14 +-
.../solr/handler/component/ExpandComponent.java | 68 +-
.../solr/handler/component/FieldFacetStats.java | 22 +-
.../solr/handler/component/StatsValues.java | 2 +-
.../handler/component/StatsValuesFactory.java | 8 +-
.../solr/index/SlowCompositeReaderWrapper.java | 39 +-
.../apache/solr/request/DocValuesFacets.java | 70 +-
.../org/apache/solr/request/DocValuesStats.java | 48 +-
.../org/apache/solr/request/IntervalFacets.java | 58 +-
.../org/apache/solr/request/NumericFacets.java | 31 +-
.../request/PerSegmentSingleValuedFaceting.java | 40 +-
.../solr/response/SortingResponseWriter.java | 151 ++-
.../transform/ValueSourceAugmenter.java | 8 +-
.../java/org/apache/solr/schema/BoolField.java | 24 +-
.../org/apache/solr/schema/CurrencyField.java | 32 +-
.../java/org/apache/solr/schema/LatLonType.java | 2 +-
.../schema/RptWithGeometrySpatialField.java | 10 +-
.../org/apache/solr/schema/StrFieldSource.java | 8 +-
.../org/apache/solr/schema/TrieDoubleField.java | 48 +-
.../org/apache/solr/schema/TrieFloatField.java | 49 +-
.../org/apache/solr/schema/TrieIntField.java | 49 +-
.../org/apache/solr/schema/TrieLongField.java | 49 +-
.../solr/search/CollapsingQParserPlugin.java | 227 +++-
.../apache/solr/search/HashQParserPlugin.java | 29 +-
.../solr/search/IGainTermsQParserPlugin.java | 13 +-
.../apache/solr/search/SolrIndexSearcher.java | 55 +-
.../TextLogisticRegressionQParserPlugin.java | 11 +-
.../apache/solr/search/ValueSourceParser.java | 78 +-
.../facet/FacetFieldProcessorByArrayDV.java | 78 +-
.../facet/FacetFieldProcessorByHashDV.java | 19 +-
.../org/apache/solr/search/facet/HLLAgg.java | 13 +-
.../apache/solr/search/facet/PercentileAgg.java | 2 +-
.../org/apache/solr/search/facet/SlotAcc.java | 12 +-
.../org/apache/solr/search/facet/UniqueAgg.java | 12 +-
.../solr/search/facet/UniqueMultiDvSlotAcc.java | 35 +-
.../search/facet/UniqueSinglevaluedSlotAcc.java | 24 +-
.../solr/search/function/OrdFieldSource.java | 33 +-
.../search/function/ReverseOrdFieldSource.java | 13 +-
.../function/SolrComparisonBoolFunction.java | 4 +-
.../function/distance/GeohashFunction.java | 4 +-
.../distance/GeohashHaversineFunction.java | 6 +-
.../distance/HaversineConstFunction.java | 4 +-
.../function/distance/HaversineFunction.java | 6 +-
.../distance/SquaredEuclideanFunction.java | 4 +-
.../distance/StringDistanceFunction.java | 6 +-
.../distance/VectorDistanceFunction.java | 6 +-
.../join/BlockJoinFieldFacetAccumulator.java | 31 +-
.../solr/search/join/GraphTermsCollector.java | 20 +-
.../apache/solr/uninverting/DocTermOrds.java | 6 +-
.../org/apache/solr/uninverting/FieldCache.java | 15 +-
.../apache/solr/uninverting/FieldCacheImpl.java | 320 ++++--
.../solr/uninverting/UninvertingReader.java | 64 +-
.../apache/solr/schema/DocValuesMultiTest.java | 10 +-
.../org/apache/solr/schema/DocValuesTest.java | 48 +-
.../test/org/apache/solr/search/TestDocSet.java | 5 -
.../apache/solr/search/TestRankQueryPlugin.java | 14 +-
.../search/function/NvlValueSourceParser.java | 6 +-
.../solr/uninverting/TestDocTermOrds.java | 44 +-
.../apache/solr/uninverting/TestFieldCache.java | 211 ++--
.../solr/uninverting/TestFieldCacheReopen.java | 10 +-
.../TestFieldCacheSanityChecker.java | 18 +-
.../uninverting/TestFieldCacheSortRandom.java | 3 +-
.../uninverting/TestFieldCacheVsDocValues.java | 55 +-
.../uninverting/TestFieldCacheWithThreads.java | 60 +-
.../solr/uninverting/TestLegacyFieldCache.java | 147 +--
.../solr/uninverting/TestUninvertingReader.java | 16 +-
.../apache/solr/update/DocumentBuilderTest.java | 9 +-
335 files changed, 10597 insertions(+), 5844 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7c60d2c..c7d71a2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -18,6 +18,10 @@ API Changes
* LUCENE-7355: AnalyzingQueryParser has been removed as its functionality has
been folded into the classic QueryParser. (Adrien Grand)
+* LUCENE-7407: Doc values APIs have been switched from random access
+ to iterators, enabling future codec compression improvements. (Mike
+ McCandless)
+
Bug Fixes
Improvements
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
index 3a81ba4..018422f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
@@ -120,7 +120,8 @@ public class TestCollationDocValuesField extends LuceneTestCase {
Document doc = is.doc(docID);
String s = doc.getField("field").stringValue();
boolean collatorAccepts = collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;
- BytesRef br = dvs.get(docID);
+ assertEquals(docID, dvs.nextDoc());
+ BytesRef br = dvs.binaryValue();
boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
assertEquals(startPoint + " <= " + s + " <= " + endPoint, collatorAccepts, luceneAccepts);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java
index 28fdd43..24862c4 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java
@@ -117,7 +117,8 @@ public class TestICUCollationDocValuesField extends LuceneTestCase {
Document doc = is.doc(docID);
String s = doc.getField("field").stringValue();
boolean collatorAccepts = collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
- BytesRef br = dvs.get(docID);
+ assertEquals(docID, dvs.nextDoc());
+ BytesRef br = dvs.binaryValue();
boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
assertEquals(collatorAccepts, luceneAccepts);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 62cd9cf..6c56cec 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -778,43 +778,56 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
for (int i=0;i<35;i++) {
int id = Integer.parseInt(reader.document(i).get("id"));
- assertEquals(id, dvByte.get(i));
+ assertEquals(i, dvByte.nextDoc());
+ assertEquals(id, dvByte.longValue());
byte bytes[] = new byte[] {
(byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id
};
BytesRef expectedRef = new BytesRef(bytes);
- BytesRef term = dvBytesDerefFixed.get(i);
+ assertEquals(i, dvBytesDerefFixed.nextDoc());
+ BytesRef term = dvBytesDerefFixed.binaryValue();
assertEquals(expectedRef, term);
- term = dvBytesDerefVar.get(i);
+ assertEquals(i, dvBytesDerefVar.nextDoc());
+ term = dvBytesDerefVar.binaryValue();
assertEquals(expectedRef, term);
- term = dvBytesSortedFixed.get(i);
+ assertEquals(i, dvBytesSortedFixed.nextDoc());
+ term = dvBytesSortedFixed.binaryValue();
assertEquals(expectedRef, term);
- term = dvBytesSortedVar.get(i);
+ assertEquals(i, dvBytesSortedVar.nextDoc());
+ term = dvBytesSortedVar.binaryValue();
assertEquals(expectedRef, term);
- term = dvBytesStraightFixed.get(i);
+ assertEquals(i, dvBytesStraightFixed.nextDoc());
+ term = dvBytesStraightFixed.binaryValue();
assertEquals(expectedRef, term);
- term = dvBytesStraightVar.get(i);
+ assertEquals(i, dvBytesStraightVar.nextDoc());
+ term = dvBytesStraightVar.binaryValue();
assertEquals(expectedRef, term);
- assertEquals((double)id, Double.longBitsToDouble(dvDouble.get(i)), 0D);
- assertEquals((float)id, Float.intBitsToFloat((int)dvFloat.get(i)), 0F);
- assertEquals(id, dvInt.get(i));
- assertEquals(id, dvLong.get(i));
- assertEquals(id, dvPacked.get(i));
- assertEquals(id, dvShort.get(i));
+ assertEquals(i, dvDouble.nextDoc());
+ assertEquals((double)id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
+ assertEquals(i, dvFloat.nextDoc());
+ assertEquals((float)id, Float.intBitsToFloat((int)dvFloat.longValue()), 0F);
+ assertEquals(i, dvInt.nextDoc());
+ assertEquals(id, dvInt.longValue());
+ assertEquals(i, dvLong.nextDoc());
+ assertEquals(id, dvLong.longValue());
+ assertEquals(i, dvPacked.nextDoc());
+ assertEquals(id, dvPacked.longValue());
+ assertEquals(i, dvShort.nextDoc());
+ assertEquals(id, dvShort.longValue());
if (is42Index) {
- dvSortedSet.setDocument(i);
+ assertEquals(i, dvSortedSet.nextDoc());
long ord = dvSortedSet.nextOrd();
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
term = dvSortedSet.lookupOrd(ord);
assertEquals(expectedRef, term);
}
if (is49Index) {
- dvSortedNumeric.setDocument(i);
- assertEquals(1, dvSortedNumeric.count());
- assertEquals(id, dvSortedNumeric.valueAt(0));
+ assertEquals(i, dvSortedNumeric.nextDoc());
+ assertEquals(1, dvSortedNumeric.docValueCount());
+ assertEquals(id, dvSortedNumeric.nextValue());
}
}
}
@@ -1378,7 +1391,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
NumericDocValues ndvf = r.getNumericDocValues(f);
NumericDocValues ndvcf = r.getNumericDocValues(cf);
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(ndvcf.get(i), ndvf.get(i)*2);
+ assertEquals(i, ndvcf.nextDoc());
+ assertEquals(i, ndvf.nextDoc());
+ assertEquals(ndvcf.longValue(), ndvf.longValue()*2);
}
}
@@ -1386,7 +1401,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
BinaryDocValues bdvf = r.getBinaryDocValues(f);
BinaryDocValues bdvcf = r.getBinaryDocValues(cf);
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(getValue(bdvcf, i), getValue(bdvf, i)*2);
+ assertEquals(i, bdvf.nextDoc());
+ assertEquals(i, bdvcf.nextDoc());
+ assertEquals(getValue(bdvcf), getValue(bdvf)*2);
}
}
@@ -1464,9 +1481,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
- static long getValue(BinaryDocValues bdv, int idx) {
- BytesRef term = bdv.get(idx);
- idx = term.offset;
+ static long getValue(BinaryDocValues bdv) {
+ BytesRef term = bdv.binaryValue();
+ int idx = term.offset;
byte b = term.bytes[idx++];
long value = b & 0x7FL;
for (int shift = 7; (b & 0x80L) != 0; shift += 7) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
index 4260745..7d096a4 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
@@ -22,6 +22,8 @@ import java.util.Iterator;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -29,14 +31,14 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.VERSION_CURRENT;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.BYTES;
+import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.NUMBER;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED_NUMERIC;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED_NUMERIC_SINGLETON;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED_SET;
import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED_SET_SINGLETON;
-import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.NUMBER;
+import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.VERSION_CURRENT;
/**
* Writer for {@link DirectDocValuesFormat}
@@ -65,10 +67,10 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
meta.writeVInt(field.number);
meta.writeByte(NUMBER);
- addNumericFieldValues(field, values);
+ addNumericFieldValues(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, maxDoc));
}
private void addNumericFieldValues(FieldInfo field, Iterable<Number> values) throws IOException {
@@ -162,10 +164,10 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
+ public void addBinaryField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
meta.writeVInt(field.number);
meta.writeByte(BYTES);
- addBinaryFieldValues(field, values);
+ addBinaryFieldValues(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, maxDoc));
}
private void addBinaryFieldValues(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
@@ -231,19 +233,22 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
meta.writeVInt(field.number);
meta.writeByte(SORTED);
// write the ordinals as numerics
- addNumericFieldValues(field, docToOrd);
-
+ addNumericFieldValues(field, LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, maxDoc));
// write the values as binary
- addBinaryFieldValues(field, values);
+ addBinaryFieldValues(field, LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field)));
}
@Override
- public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
+
+ final Iterable<Number> docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, maxDoc);
+ final Iterable<Number> values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field);
+
meta.writeVInt(field.number);
if (isSingleValued(docToValueCount)) {
meta.writeByte(SORTED_NUMERIC_SINGLETON);
@@ -264,7 +269,11 @@ class DirectDocValuesConsumer extends DocValuesConsumer {
// note: this might not be the most efficient... but it's fairly simple
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field));
+ Iterable<Number> docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, maxDoc);
+ Iterable<Number> ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field);
+
meta.writeVInt(field.number);
if (isSingleValued(docToOrdCount)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
index 1dfd2f2..fe6c8f6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
@@ -28,18 +28,8 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -284,15 +274,16 @@ class DirectDocValuesProducer extends DocValuesProducer {
@Override
public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericRawValues instance = numericInstances.get(field.name);
+ NumericEntry ne = numerics.get(field.name);
if (instance == null) {
// Lazy load
- instance = loadNumeric(numerics.get(field.name));
+ instance = loadNumeric(ne);
if (!merging) {
numericInstances.put(field.name, instance);
ramBytesUsed.addAndGet(instance.ramBytesUsed());
}
}
- return instance.numerics;
+ return new LegacyNumericDocValuesWrapper(getMissingBits(field, ne.missingOffset, ne.missingBytes), instance.numerics);
}
private NumericRawValues loadNumeric(NumericEntry entry) throws IOException {
@@ -305,7 +296,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
final byte[] values = new byte[entry.count];
data.readBytes(values, 0, entry.count);
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ret.numerics = new NumericDocValues() {
+ ret.numerics = new LegacyNumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
@@ -321,7 +312,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
values[i] = data.readShort();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ret.numerics = new NumericDocValues() {
+ ret.numerics = new LegacyNumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
@@ -337,7 +328,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
values[i] = data.readInt();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ret.numerics = new NumericDocValues() {
+ ret.numerics = new LegacyNumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
@@ -353,7 +344,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
values[i] = data.readLong();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ret.numerics = new NumericDocValues() {
+ ret.numerics = new LegacyNumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
@@ -367,8 +358,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
}
- @Override
- public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ private synchronized LegacyBinaryDocValues getLegacyBinary(FieldInfo field) throws IOException {
BinaryRawValues instance = binaryInstances.get(field.name);
if (instance == null) {
// Lazy load
@@ -381,7 +371,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
final byte[] bytes = instance.bytes;
final int[] address = instance.address;
- return new BinaryDocValues() {
+ return new LegacyBinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
@@ -394,6 +384,12 @@ class DirectDocValuesProducer extends DocValuesProducer {
};
}
+ @Override
+ public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ BinaryEntry be = binaries.get(field.name);
+ return new LegacyBinaryDocValuesWrapper(getMissingBits(field, be.missingOffset, be.missingBytes), getLegacyBinary(field));
+ }
+
private BinaryRawValues loadBinary(BinaryEntry entry) throws IOException {
IndexInput data = this.data.clone();
data.seek(entry.offset);
@@ -428,11 +424,11 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
}
}
- return newSortedInstance(instance.docToOrd.numerics, getBinary(field), entry.values.count);
+ return new LegacySortedDocValuesWrapper(newSortedInstance(instance.docToOrd.numerics, getLegacyBinary(field), entry.values.count), maxDoc);
}
- private SortedDocValues newSortedInstance(final NumericDocValues docToOrd, final BinaryDocValues values, final int count) {
- return new SortedDocValues() {
+ private LegacySortedDocValues newSortedInstance(final LegacyNumericDocValues docToOrd, final LegacyBinaryDocValues values, final int count) {
+ return new LegacySortedDocValues() {
@Override
public int getOrd(int docID) {
@@ -477,14 +473,14 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
if (entry.docToAddress == null) {
- final NumericDocValues single = instance.values.numerics;
+ final LegacyNumericDocValues single = instance.values.numerics;
final Bits docsWithField = getMissingBits(field, entry.values.missingOffset, entry.values.missingBytes);
- return DocValues.singleton(single, docsWithField);
+ return DocValues.singleton(new LegacyNumericDocValuesWrapper(docsWithField, single));
} else {
- final NumericDocValues docToAddress = instance.docToAddress.numerics;
- final NumericDocValues values = instance.values.numerics;
+ final LegacyNumericDocValues docToAddress = instance.docToAddress.numerics;
+ final LegacyNumericDocValues values = instance.values.numerics;
- return new SortedNumericDocValues() {
+ return new LegacySortedNumericDocValuesWrapper(new LegacySortedNumericDocValues() {
int valueStart;
int valueLimit;
@@ -503,7 +499,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
public int count() {
return valueLimit - valueStart;
}
- };
+ }, maxDoc);
}
}
@@ -530,15 +526,15 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
if (instance.docToOrdAddress == null) {
- SortedDocValues sorted = newSortedInstance(instance.ords.numerics, getBinary(field), entry.values.count);
- return DocValues.singleton(sorted);
+ LegacySortedDocValues sorted = newSortedInstance(instance.ords.numerics, getLegacyBinary(field), entry.values.count);
+ return DocValues.singleton(new LegacySortedDocValuesWrapper(sorted, maxDoc));
} else {
- final NumericDocValues docToOrdAddress = instance.docToOrdAddress.numerics;
- final NumericDocValues ords = instance.ords.numerics;
- final BinaryDocValues values = getBinary(field);
+ final LegacyNumericDocValues docToOrdAddress = instance.docToOrdAddress.numerics;
+ final LegacyNumericDocValues ords = instance.ords.numerics;
+ final LegacyBinaryDocValues values = getLegacyBinary(field);
// Must make a new instance since the iterator has state:
- return new RandomAccessOrds() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
int ordStart;
int ordUpto;
int ordLimit;
@@ -568,20 +564,10 @@ class DirectDocValuesProducer extends DocValuesProducer {
return entry.values.count;
}
- @Override
- public long ordAt(int index) {
- return ords.get(ordStart + index);
- }
-
- @Override
- public int cardinality() {
- return ordLimit - ordStart;
- }
-
// Leave lookupTerm to super's binary search
// Leave termsEnum to super
- };
+ }, maxDoc);
}
}
@@ -621,26 +607,6 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- switch(field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED_NUMERIC:
- return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- BinaryEntry be = binaries.get(field.name);
- return getMissingBits(field, be.missingOffset, be.missingBytes);
- case NUMERIC:
- NumericEntry ne = numerics.get(field.name);
- return getMissingBits(field, ne.missingOffset, ne.missingBytes);
- default:
- throw new AssertionError();
- }
- }
-
- @Override
public synchronized DocValuesProducer getMergeInstance() throws IOException {
return new DirectDocValuesProducer(this);
}
@@ -680,7 +646,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
static class NumericRawValues implements Accountable {
- NumericDocValues numerics;
+ LegacyNumericDocValues numerics;
long bytesUsed;
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
index 44ffdf9..eb7d096 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
@@ -25,6 +25,8 @@ import java.util.NoSuchElementException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -45,19 +47,19 @@ import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.VERSION_CURRENT;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_COMPRESSED;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_SIZE;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BYTES;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.NUMBER;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.DELTA_COMPRESSED;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.FST;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET_SINGLETON;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.GCD_COMPRESSED;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.NUMBER;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_NUMERIC;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_NUMERIC_SINGLETON;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.DELTA_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.GCD_COMPRESSED;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET_SINGLETON;
import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.TABLE_COMPRESSED;
+import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.VERSION_CURRENT;
/**
* Writer for {@link MemoryDocValuesFormat}
@@ -87,8 +89,8 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- addNumericField(field, values, true);
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addNumericField(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, maxDoc), true);
}
void addNumericField(FieldInfo field, Iterable<Number> values, boolean optimizeStorage) throws IOException {
@@ -297,7 +299,11 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addBinaryField(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, maxDoc));
+ }
+
+ private void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
// write the byte[] data
meta.writeVInt(field.number);
meta.writeByte(BYTES);
@@ -396,7 +402,13 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addSortedField(field,
+ LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field)),
+ LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, maxDoc));
+ }
+
+ private void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
// write the ordinals as numerics
addNumericField(field, docToOrd, false);
@@ -405,7 +417,11 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
}
@Override
- public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
+
+ final Iterable<Number> docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, maxDoc);
+ final Iterable<Number> values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field);
+
meta.writeVInt(field.number);
if (isSingleValued(docToValueCount)) {
@@ -436,7 +452,10 @@ class MemoryDocValuesConsumer extends DocValuesConsumer {
// note: this might not be the most efficient... but it's fairly simple
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field));
+ Iterable<Number> docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, maxDoc);
+ Iterable<Number> ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field);
meta.writeVInt(field.number);
if (isSingleValued(docToOrdCount)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
index 46cf3e5..bff8c2d 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
@@ -28,19 +28,8 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -56,11 +45,11 @@ import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
-import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
+import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.packed.BlockPackedReader;
@@ -80,7 +69,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
private final IndexInput data;
// ram instances we have already loaded
- private final Map<String,NumericDocValues> numericInstances = new HashMap<>();
+ private final Map<String,LegacyNumericDocValues> numericInstances = new HashMap<>();
private final Map<String,BytesAndAddresses> pagedBytesInstances = new HashMap<>();
private final Map<String,FST<Long>> fstInstances = new HashMap<>();
private final Map<String,FixedBitSet> docsWithFieldInstances = new HashMap<>();
@@ -280,7 +269,19 @@ class MemoryDocValuesProducer extends DocValuesProducer {
@Override
public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericDocValues instance = numericInstances.get(field.name);
+ LegacyNumericDocValues instance = numericInstances.get(field.name);
+ if (instance == null) {
+ instance = loadNumeric(field);
+ if (!merging) {
+ numericInstances.put(field.name, instance);
+ }
+ }
+ NumericEntry ne = numerics.get(field.name);
+ return new LegacyNumericDocValuesWrapper(getMissingBits(field, ne.missingOffset, ne.missingBytes), instance);
+ }
+
+ private synchronized LegacyNumericDocValues getNumericNonIterator(FieldInfo field) throws IOException {
+ LegacyNumericDocValues instance = numericInstances.get(field.name);
if (instance == null) {
instance = loadNumeric(field);
if (!merging) {
@@ -321,7 +322,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
return getClass().getSimpleName() + "(entries=" + numEntries + ")";
}
- private NumericDocValues loadNumeric(FieldInfo field) throws IOException {
+ private LegacyNumericDocValues loadNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.name);
IndexInput data = this.data.clone();
data.seek(entry.offset + entry.missingBytes);
@@ -342,7 +343,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
numericInfo.put(field.name, Accountables.namedAccountable("table compressed", ordsReader));
}
- return new NumericDocValues() {
+ return new LegacyNumericDocValues() {
@Override
public long get(int docID) {
return decode[(int)ordsReader.get(docID)];
@@ -357,7 +358,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
ramBytesUsed.addAndGet(deltaReader.ramBytesUsed());
numericInfo.put(field.name, Accountables.namedAccountable("delta compressed", deltaReader));
}
- return new NumericDocValues() {
+ return new LegacyNumericDocValues() {
@Override
public long get(int docID) {
return minDelta + deltaReader.get(docID);
@@ -381,7 +382,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
numericInfo.put(field.name, Accountables.namedAccountable("gcd compressed", quotientReader));
}
- return new NumericDocValues() {
+ return new LegacyNumericDocValues() {
@Override
public long get(int docID) {
return min + mult * quotientReader.get(docID);
@@ -392,8 +393,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
}
- @Override
- public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ private LegacyBinaryDocValues getLegacyBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.name);
BytesAndAddresses instance;
@@ -412,7 +412,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
if (addresses == null) {
assert entry.minLength == entry.maxLength;
final int fixedLength = entry.minLength;
- return new BinaryDocValues() {
+ return new LegacyBinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
@@ -422,7 +422,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
};
} else {
- return new BinaryDocValues() {
+ return new LegacyBinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
@@ -436,6 +436,12 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
}
+ @Override
+ public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ BinaryEntry be = binaries.get(field.name);
+ return new LegacyBinaryDocValuesWrapper(getMissingBits(field, be.missingOffset, be.missingBytes), getLegacyBinary(field));
+ }
+
private BytesAndAddresses loadBinary(FieldInfo field) throws IOException {
BytesAndAddresses bytesAndAddresses = new BytesAndAddresses();
BinaryEntry entry = binaries.get(field.name);
@@ -459,9 +465,13 @@ class MemoryDocValuesProducer extends DocValuesProducer {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ return new LegacySortedDocValuesWrapper(getSortedNonIterator(field), maxDoc);
+ }
+
+ private LegacySortedDocValues getSortedNonIterator(FieldInfo field) throws IOException {
final FSTEntry entry = fsts.get(field.name);
if (entry.numOrds == 0) {
- return DocValues.emptySorted();
+ return DocValues.emptyLegacySorted();
}
FST<Long> instance;
synchronized(this) {
@@ -476,7 +486,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
}
}
- final NumericDocValues docToOrd = getNumeric(field);
+ final LegacyNumericDocValues docToOrd = getNumericNonIterator(field);
final FST<Long> fst = instance;
// per-thread resources
@@ -486,7 +496,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
final IntsRefBuilder scratchInts = new IntsRefBuilder();
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
- return new SortedDocValues() {
+ return new LegacySortedDocValues() {
final BytesRefBuilder term = new BytesRefBuilder();
@Override
@@ -538,12 +548,12 @@ class MemoryDocValuesProducer extends DocValuesProducer {
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.name);
if (entry.singleton) {
- NumericDocValues values = getNumeric(field);
+ LegacyNumericDocValues values = getNumericNonIterator(field);
NumericEntry ne = numerics.get(field.name);
Bits docsWithField = getMissingBits(field, ne.missingOffset, ne.missingBytes);
- return DocValues.singleton(values, docsWithField);
+ return DocValues.singleton(new LegacyNumericDocValuesWrapper(docsWithField, values));
} else {
- final NumericDocValues values = getNumeric(field);
+ final LegacyNumericDocValues values = getNumericNonIterator(field);
final MonotonicBlockPackedReader addr;
synchronized (this) {
MonotonicBlockPackedReader res = addresses.get(field.name);
@@ -561,7 +571,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
if (values instanceof LongValues) {
// probably not the greatest codec choice for this situation, but we support it
final LongValues longValues = (LongValues) values;
- return new SortedNumericDocValues() {
+ return new LegacySortedNumericDocValuesWrapper(new LegacySortedNumericDocValues() {
long startOffset;
long endOffset;
@@ -580,9 +590,9 @@ class MemoryDocValuesProducer extends DocValuesProducer {
public int count() {
return (int) (endOffset - startOffset);
}
- };
+ }, maxDoc);
} else {
- return new SortedNumericDocValues() {
+ return new LegacySortedNumericDocValuesWrapper(new LegacySortedNumericDocValues() {
int startOffset;
int endOffset;
@@ -601,7 +611,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
public int count() {
return (endOffset - startOffset);
}
- };
+ }, maxDoc);
}
}
}
@@ -630,7 +640,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
}
}
- final BinaryDocValues docToOrds = getBinary(field);
+ final LegacyBinaryDocValues docToOrds = getLegacyBinary(field);
final FST<Long> fst = instance;
// per-thread resources
@@ -640,7 +650,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
final IntsRefBuilder scratchInts = new IntsRefBuilder();
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
final ByteArrayDataInput input = new ByteArrayDataInput();
- return new SortedSetDocValues() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
final BytesRefBuilder term = new BytesRefBuilder();
BytesRef ref;
long currentOrd;
@@ -699,7 +709,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
public TermsEnum termsEnum() {
return new FSTTermsEnum(fst);
}
- };
+ }, maxDoc);
}
private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
@@ -729,26 +739,6 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- switch(field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED_NUMERIC:
- return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- BinaryEntry be = binaries.get(field.name);
- return getMissingBits(field, be.missingOffset, be.missingBytes);
- case NUMERIC:
- NumericEntry ne = numerics.get(field.name);
- return getMissingBits(field, ne.missingOffset, ne.missingBytes);
- default:
- throw new AssertionError();
- }
- }
-
- @Override
public void close() throws IOException {
data.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
index 8b97128..c0b88cc 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
@@ -29,17 +29,8 @@ import java.util.Locale;
import java.util.Map;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.BufferedChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -148,6 +139,15 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
@Override
public NumericDocValues getNumeric(FieldInfo fieldInfo) throws IOException {
+ LegacyNumericDocValues values = getNumericNonIterator(fieldInfo);
+ if (values == null) {
+ return null;
+ } else {
+ return new LegacyNumericDocValuesWrapper(getNumericDocsWithField(fieldInfo), values);
+ }
+ }
+
+ LegacyNumericDocValues getNumericNonIterator(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
assert field != null;
@@ -161,7 +161,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
decoder.setParseBigDecimal(true);
- return new NumericDocValues() {
+ return new LegacyNumericDocValues() {
@Override
public long get(int docID) {
try {
@@ -211,8 +211,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
};
}
- @Override
- public BinaryDocValues getBinary(FieldInfo fieldInfo) throws IOException {
+ private LegacyBinaryDocValues getLegacyBinary(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
// SegmentCoreReaders already verifies this field is
@@ -223,7 +222,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
final BytesRefBuilder scratch = new BytesRefBuilder();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
- return new BinaryDocValues() {
+ return new LegacyBinaryDocValues() {
final BytesRefBuilder term = new BytesRefBuilder();
@Override
@@ -252,6 +251,11 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
};
}
+ @Override
+ public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ return new LegacyBinaryDocValuesWrapper(getBinaryDocsWithField(field), getLegacyBinary(field));
+ }
+
private Bits getBinaryDocsWithField(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
final IndexInput in = data.clone();
@@ -302,7 +306,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));
- return new SortedDocValues() {
+ return new LegacySortedDocValuesWrapper(new LegacySortedDocValues() {
final BytesRefBuilder term = new BytesRefBuilder();
@Override
@@ -351,13 +355,13 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
public int getValueCount() {
return (int)field.numValues;
}
- };
+ }, maxDoc);
}
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- final BinaryDocValues binary = getBinary(field);
- return new SortedNumericDocValues() {
+ final LegacyBinaryDocValues binary = getLegacyBinary(field);
+ return new LegacySortedNumericDocValuesWrapper(new LegacySortedNumericDocValues() {
long values[];
@Override
@@ -383,7 +387,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
public int count() {
return values.length;
}
- };
+ }, maxDoc);
}
@Override
@@ -398,7 +402,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
final BytesRefBuilder scratch = new BytesRefBuilder();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
- return new SortedSetDocValues() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
String[] currentOrds = new String[0];
int currentIndex = 0;
final BytesRefBuilder term = new BytesRefBuilder();
@@ -460,28 +464,10 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
public long getValueCount() {
return field.numValues;
}
- };
+ }, maxDoc);
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- switch (field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED_NUMERIC:
- return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- return getBinaryDocsWithField(field);
- case NUMERIC:
- return getNumericDocsWithField(field);
- default:
- throw new AssertionError();
- }
- }
-
- @Override
public void close() throws IOException {
data.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
index 0d28479..379942e 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
@@ -27,10 +27,12 @@ import java.util.Locale;
import java.util.Set;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -69,7 +71,12 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addNumericField(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, numDocs));
+ }
+
+ void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+
assert fieldSeen(field.name);
assert field.getDocValuesType() == DocValuesType.NUMERIC || field.hasNorms();
writeFieldEntry(field, DocValuesType.NUMERIC);
@@ -133,10 +140,10 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
}
@Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
assert fieldSeen(field.name);
assert field.getDocValuesType() == DocValuesType.BINARY;
- doAddBinary(field, values);
+ doAddBinary(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, numDocs));
}
private void doAddBinary(FieldInfo field, Iterable<BytesRef> values) throws IOException {
@@ -195,7 +202,10 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field));
+ Iterable<Number> docToOrd = LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, numDocs);
+
assert fieldSeen(field.name);
assert field.getDocValuesType() == DocValuesType.SORTED;
writeFieldEntry(field, DocValuesType.SORTED);
@@ -272,7 +282,11 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
}
@Override
- public void addSortedNumericField(FieldInfo field, final Iterable<Number> docToValueCount, final Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
+
+ final Iterable<Number> docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, numDocs);
+ final Iterable<Number> values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field);
+
assert fieldSeen(field.name);
assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
doAddBinary(field, new Iterable<BytesRef>() {
@@ -314,7 +328,10 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
}
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field));
+ Iterable<Number> docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, numDocs);
+ Iterable<Number> ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field);
assert fieldSeen(field.name);
assert field.getDocValuesType() == DocValuesType.SORTED_SET;
writeFieldEntry(field, DocValuesType.SORTED_SET);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
index 53e9b87..faa50b7 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
@@ -20,14 +20,17 @@ package org.apache.lucene.codecs.simpletext;
import java.io.IOException;
import java.util.Collection;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LegacyNumericDocValuesWrapper;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Bits;
/**
* plain-text norms format.
@@ -67,7 +70,7 @@ public class SimpleTextNormsFormat extends NormsFormat {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
- return impl.getNumeric(field);
+ return new LegacyNumericDocValuesWrapper(new Bits.MatchAllBits(impl.maxDoc), impl.getNumericNonIterator(field));
}
@Override
@@ -113,8 +116,8 @@ public class SimpleTextNormsFormat extends NormsFormat {
}
@Override
- public void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
- impl.addNumericField(field, values);
+ public void addNormsField(FieldInfo field, NormsProducer normsProducer) throws IOException {
+ impl.addNumericField(field, LegacyDocValuesIterables.normsIterable(field, normsProducer, impl.numDocs));
}
@Override
[15/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
index 427b520..87c236c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
@@ -21,12 +21,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.NoSuchElementException;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocIDMerger;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.EmptyDocValuesProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.MergeState;
@@ -37,7 +37,6 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongBitSet;
@@ -77,51 +76,42 @@ public abstract class DocValuesConsumer implements Closeable {
/**
* Writes numeric docvalues for a field.
* @param field field information
- * @param values Iterable of numeric values (one for each document). {@code null} indicates
- * a missing value.
+ * @param valuesProducer Numeric values to write.
* @throws IOException if an I/O error occurred.
*/
- public abstract void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException;
+ public abstract void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException;
/**
* Writes binary docvalues for a field.
* @param field field information
- * @param values Iterable of binary values (one for each document). {@code null} indicates
- * a missing value.
+ * @param valuesProducer Binary values to write.
* @throws IOException if an I/O error occurred.
*/
- public abstract void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException;
+ public abstract void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException;
/**
* Writes pre-sorted binary docvalues for a field.
* @param field field information
- * @param values Iterable of binary values in sorted order (deduplicated).
- * @param docToOrd Iterable of ordinals (one for each document). {@code -1} indicates
- * a missing value.
+ * @param valuesProducer produces the values and ordinals to write
* @throws IOException if an I/O error occurred.
*/
- public abstract void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException;
+ public abstract void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException;
/**
* Writes pre-sorted numeric docvalues for a field
* @param field field information
- * @param docToValueCount Iterable of the number of values for each document. A zero
- * count indicates a missing value.
- * @param values Iterable of numeric values in sorted order (not deduplicated).
+ * @param valuesProducer produces the values to write
* @throws IOException if an I/O error occurred.
*/
- public abstract void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException;
+ public abstract void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException;
/**
* Writes pre-sorted set docvalues for a field
* @param field field information
- * @param values Iterable of binary values in sorted order (deduplicated).
- * @param docToOrdCount Iterable of the number of values for each document. A zero ordinal
- * count indicates a missing value.
- * @param ords Iterable of ordinal occurrences (docToOrdCount*maxDoc total).
+ * @param valuesProducer produces the values to write
* @throws IOException if an I/O error occurred.
*/
- public abstract void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException;
+ public abstract void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException;
/** Merges in the fields from the readers in
* <code>mergeState</code>. The default implementation
@@ -142,100 +132,15 @@ public abstract class DocValuesConsumer implements Closeable {
DocValuesType type = mergeFieldInfo.getDocValuesType();
if (type != DocValuesType.NONE) {
if (type == DocValuesType.NUMERIC) {
- List<NumericDocValues> toMerge = new ArrayList<>();
- List<Bits> docsWithField = new ArrayList<>();
- for (int i=0;i<mergeState.docValuesProducers.length;i++) {
- NumericDocValues values = null;
- Bits bits = null;
- DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
- if (docValuesProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.NUMERIC) {
- values = docValuesProducer.getNumeric(fieldInfo);
- bits = docValuesProducer.getDocsWithField(fieldInfo);
- }
- }
- if (values == null) {
- values = DocValues.emptyNumeric();
- bits = new Bits.MatchNoBits(mergeState.maxDocs[i]);
- }
- toMerge.add(values);
- docsWithField.add(bits);
- }
- mergeNumericField(mergeFieldInfo, mergeState, toMerge, docsWithField);
+ mergeNumericField(mergeFieldInfo, mergeState);
} else if (type == DocValuesType.BINARY) {
- List<BinaryDocValues> toMerge = new ArrayList<>();
- List<Bits> docsWithField = new ArrayList<>();
- for (int i=0;i<mergeState.docValuesProducers.length;i++) {
- BinaryDocValues values = null;
- Bits bits = null;
- DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
- if (docValuesProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
- values = docValuesProducer.getBinary(fieldInfo);
- bits = docValuesProducer.getDocsWithField(fieldInfo);
- }
- }
- if (values == null) {
- values = DocValues.emptyBinary();
- bits = new Bits.MatchNoBits(mergeState.maxDocs[i]);
- }
- toMerge.add(values);
- docsWithField.add(bits);
- }
- mergeBinaryField(mergeFieldInfo, mergeState, toMerge, docsWithField);
+ mergeBinaryField(mergeFieldInfo, mergeState);
} else if (type == DocValuesType.SORTED) {
- List<SortedDocValues> toMerge = new ArrayList<>();
- for (int i=0;i<mergeState.docValuesProducers.length;i++) {
- SortedDocValues values = null;
- DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
- if (docValuesProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED) {
- values = docValuesProducer.getSorted(fieldInfo);
- }
- }
- if (values == null) {
- values = DocValues.emptySorted();
- }
- toMerge.add(values);
- }
- mergeSortedField(mergeFieldInfo, mergeState, toMerge);
+ mergeSortedField(mergeFieldInfo, mergeState);
} else if (type == DocValuesType.SORTED_SET) {
- List<SortedSetDocValues> toMerge = new ArrayList<>();
- for (int i=0;i<mergeState.docValuesProducers.length;i++) {
- SortedSetDocValues values = null;
- DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
- if (docValuesProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_SET) {
- values = docValuesProducer.getSortedSet(fieldInfo);
- }
- }
- if (values == null) {
- values = DocValues.emptySortedSet();
- }
- toMerge.add(values);
- }
- mergeSortedSetField(mergeFieldInfo, mergeState, toMerge);
+ mergeSortedSetField(mergeFieldInfo, mergeState);
} else if (type == DocValuesType.SORTED_NUMERIC) {
- List<SortedNumericDocValues> toMerge = new ArrayList<>();
- for (int i=0;i<mergeState.docValuesProducers.length;i++) {
- SortedNumericDocValues values = null;
- DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
- if (docValuesProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC) {
- values = docValuesProducer.getSortedNumeric(fieldInfo);
- }
- }
- if (values == null) {
- values = DocValues.emptySortedNumeric(mergeState.maxDocs[i]);
- }
- toMerge.add(values);
- }
- mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge);
+ mergeSortedNumericField(mergeFieldInfo, mergeState);
} else {
throw new AssertionError("type=" + type);
}
@@ -246,84 +151,90 @@ public abstract class DocValuesConsumer implements Closeable {
/** Tracks state of one numeric sub-reader that we are merging */
private static class NumericDocValuesSub extends DocIDMerger.Sub {
- private final NumericDocValues values;
- private final Bits docsWithField;
- private int docID = -1;
- private final int maxDoc;
+ final NumericDocValues values;
- public NumericDocValuesSub(MergeState.DocMap docMap, NumericDocValues values, Bits docsWithField, int maxDoc) {
+ public NumericDocValuesSub(MergeState.DocMap docMap, NumericDocValues values) {
super(docMap);
this.values = values;
- this.docsWithField = docsWithField;
- this.maxDoc = maxDoc;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
}
/**
- * Merges the numeric docvalues from <code>toMerge</code>.
+ * Merges the numeric docvalues from <code>MergeState</code>.
* <p>
* The default implementation calls {@link #addNumericField}, passing
- * an Iterable that merges and filters deleted documents on the fly.
+ * a DocValuesProducer that merges and filters deleted documents on the fly.
*/
- public void mergeNumericField(final FieldInfo fieldInfo, final MergeState mergeState, final List<NumericDocValues> toMerge, final List<Bits> docsWithField) throws IOException {
- addNumericField(fieldInfo,
- new Iterable<Number>() {
+ public void mergeNumericField(final FieldInfo mergeFieldInfo, final MergeState mergeState) throws IOException {
+ addNumericField(mergeFieldInfo,
+ new EmptyDocValuesProducer() {
@Override
- public Iterator<Number> iterator() {
+ public NumericDocValues getNumeric(FieldInfo fieldInfo) throws IOException {
+ if (fieldInfo != mergeFieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
- // We must make a new DocIDMerger for each iterator:
List<NumericDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new NumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), docsWithField.get(i), mergeState.maxDocs[i]));
+ assert mergeState.docMaps.length == mergeState.docValuesProducers.length;
+ long cost = 0;
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ NumericDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.NUMERIC) {
+ values = docValuesProducer.getNumeric(readerFieldInfo);
+ }
+ }
+ if (values != null) {
+ cost += values.cost();
+ subs.add(new NumericDocValuesSub(mergeState.docMaps[i], values));
+ }
}
final DocIDMerger<NumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
- return new Iterator<Number>() {
- long nextValue;
- boolean nextHasValue;
- boolean nextIsSet;
+ final long finalCost = cost;
+
+ return new NumericDocValues() {
+ private int docID = -1;
+ private NumericDocValuesSub current;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
@Override
- public boolean hasNext() {
- return nextIsSet || setNext();
+ public int nextDoc() throws IOException {
+ current = docIDMerger.next();
+ if (current == null) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = current.mappedDocID;
+ }
+ return docID;
}
@Override
- public void remove() {
+ public int advance(int target) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- return nextHasValue ? nextValue : null;
+ public long cost() {
+ return finalCost;
}
- private boolean setNext() {
- NumericDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- nextIsSet = true;
- nextValue = sub.values.get(sub.docID);
- nextHasValue = nextValue != 0 || sub.docsWithField.get(sub.docID);
- return true;
+ @Override
+ public long longValue() {
+ return current.values.longValue();
}
};
}
@@ -333,90 +244,90 @@ public abstract class DocValuesConsumer implements Closeable {
/** Tracks state of one binary sub-reader that we are merging */
private static class BinaryDocValuesSub extends DocIDMerger.Sub {
- private final BinaryDocValues values;
- private final Bits docsWithField;
- private int docID = -1;
- private final int maxDoc;
+ final BinaryDocValues values;
- public BinaryDocValuesSub(MergeState.DocMap docMap, BinaryDocValues values, Bits docsWithField, int maxDoc) {
+ public BinaryDocValuesSub(MergeState.DocMap docMap, BinaryDocValues values) {
super(docMap);
this.values = values;
- this.docsWithField = docsWithField;
- this.maxDoc = maxDoc;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
}
/**
- * Merges the binary docvalues from <code>toMerge</code>.
+ * Merges the binary docvalues from <code>MergeState</code>.
* <p>
* The default implementation calls {@link #addBinaryField}, passing
- * an Iterable that merges and filters deleted documents on the fly.
+ * a DocValuesProducer that merges and filters deleted documents on the fly.
*/
- public void mergeBinaryField(FieldInfo fieldInfo, final MergeState mergeState, final List<BinaryDocValues> toMerge, final List<Bits> docsWithField) throws IOException {
- addBinaryField(fieldInfo,
- new Iterable<BytesRef>() {
+ public void mergeBinaryField(FieldInfo mergeFieldInfo, final MergeState mergeState) throws IOException {
+ addBinaryField(mergeFieldInfo,
+ new EmptyDocValuesProducer() {
@Override
- public Iterator<BytesRef> iterator() {
-
- // We must make a new DocIDMerger for each iterator:
+ public BinaryDocValues getBinary(FieldInfo fieldInfo) throws IOException {
+ if (fieldInfo != mergeFieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+
List<BinaryDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new BinaryDocValuesSub(mergeState.docMaps[i], toMerge.get(i), docsWithField.get(i), mergeState.maxDocs[i]));
+
+ long cost = 0;
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ BinaryDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ values = docValuesProducer.getBinary(readerFieldInfo);
+ }
+ }
+ if (values != null) {
+ cost += values.cost();
+ subs.add(new BinaryDocValuesSub(mergeState.docMaps[i], values));
+ }
}
final DocIDMerger<BinaryDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
+ final long finalCost = cost;
+
+ return new BinaryDocValues() {
+ private BinaryDocValuesSub current;
+ private int docID = -1;
- return new Iterator<BytesRef>() {
- BytesRef nextValue;
- BytesRef nextPointer; // points to null if missing, or nextValue
- boolean nextIsSet;
+ @Override
+ public int docID() {
+ return docID;
+ }
@Override
- public boolean hasNext() {
- return nextIsSet || setNext();
+ public int nextDoc() throws IOException {
+ current = docIDMerger.next();
+ if (current == null) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = current.mappedDocID;
+ }
+ return docID;
}
@Override
- public void remove() {
+ public int advance(int target) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public BytesRef next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- return nextPointer;
+ public long cost() {
+ return finalCost;
}
- private boolean setNext() {
- while (true) {
- BinaryDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- nextIsSet = true;
- if (sub.docsWithField.get(sub.docID)) {
- nextPointer = nextValue = sub.values.get(sub.docID);
- } else {
- nextPointer = null;
- }
- return true;
- }
- }
+ @Override
+ public BytesRef binaryValue() {
+ return current.values.binaryValue();
+ }
};
}
});
@@ -425,25 +336,17 @@ public abstract class DocValuesConsumer implements Closeable {
/** Tracks state of one sorted numeric sub-reader that we are merging */
private static class SortedNumericDocValuesSub extends DocIDMerger.Sub {
- private final SortedNumericDocValues values;
- private int docID = -1;
- private final int maxDoc;
+ final SortedNumericDocValues values;
- public SortedNumericDocValuesSub(MergeState.DocMap docMap, SortedNumericDocValues values, int maxDoc) {
+ public SortedNumericDocValuesSub(MergeState.DocMap docMap, SortedNumericDocValues values) {
super(docMap);
this.values = values;
- this.maxDoc = maxDoc;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- values.setDocument(docID);
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
}
@@ -453,149 +356,110 @@ public abstract class DocValuesConsumer implements Closeable {
* The default implementation calls {@link #addSortedNumericField}, passing
* iterables that filter deleted documents.
*/
- public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge) throws IOException {
+ public void mergeSortedNumericField(FieldInfo mergeFieldInfo, final MergeState mergeState) throws IOException {
- addSortedNumericField(fieldInfo,
- // doc -> value count
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
+ addSortedNumericField(mergeFieldInfo,
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo fieldInfo) {
+ if (fieldInfo != mergeFieldInfo) {
+ throw new IllegalArgumentException("wrong FieldInfo");
+ }
+
+ // We must make new iterators + DocIDMerger for each iterator:
+ List<SortedNumericDocValuesSub> subs = new ArrayList<>();
+ long cost = 0;
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ SortedNumericDocValues values = null;
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC) {
+ try {
+ values = docValuesProducer.getSortedNumeric(readerFieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+ if (values == null) {
+ values = DocValues.emptySortedNumeric(mergeState.maxDocs[i]);
+ }
+ cost += values.cost();
+ subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], values));
+ }
- // We must make a new DocIDMerger for each iterator:
- List<SortedNumericDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i]));
- }
+ final long finalCost = cost;
- final DocIDMerger<SortedNumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
-
- return new Iterator<Number>() {
- int nextValue;
- boolean nextIsSet;
-
- @Override
- public boolean hasNext() {
- return nextIsSet || setNext();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- return nextValue;
- }
-
- private boolean setNext() {
- while (true) {
- SortedNumericDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- nextIsSet = true;
- nextValue = sub.values.count();
- return true;
- }
- }
- };
- }
- },
- // values
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- // We must make a new DocIDMerger for each iterator:
- List<SortedNumericDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i]));
- }
+ final DocIDMerger<SortedNumericDocValuesSub> docIDMerger;
+ try {
+ docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
- final DocIDMerger<SortedNumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
-
- return new Iterator<Number>() {
- long nextValue;
- boolean nextIsSet;
- int valueUpto;
- int valueLength;
- SortedNumericDocValuesSub current;
-
- @Override
- public boolean hasNext() {
- return nextIsSet || setNext();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- return nextValue;
- }
-
- private boolean setNext() {
- while (true) {
-
- if (valueUpto < valueLength) {
- nextValue = current.values.valueAt(valueUpto);
- valueUpto++;
- nextIsSet = true;
- return true;
- }
-
- current = docIDMerger.next();
- if (current == null) {
- return false;
- }
- valueUpto = 0;
- valueLength = current.values.count();
- continue;
- }
- }
- };
- }
- }
- );
+ return new SortedNumericDocValues() {
+
+ private int docID = -1;
+ private SortedNumericDocValuesSub currentSub;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ currentSub = docIDMerger.next();
+ if (currentSub == null) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = currentSub.mappedDocID;
+ }
+
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int docValueCount() {
+ return currentSub.values.docValueCount();
+ }
+
+ @Override
+ public long cost() {
+ return finalCost;
+ }
+
+ @Override
+ public long nextValue() throws IOException {
+ return currentSub.values.nextValue();
+ }
+ };
+ }
+ });
}
/** Tracks state of one sorted sub-reader that we are merging */
private static class SortedDocValuesSub extends DocIDMerger.Sub {
- private final SortedDocValues values;
- private int docID = -1;
- private final int maxDoc;
- private final LongValues map;
-
- public SortedDocValuesSub(MergeState.DocMap docMap, SortedDocValues values, int maxDoc, LongValues map) {
+ final SortedDocValues values;
+ final LongValues map;
+
+ public SortedDocValuesSub(MergeState.DocMap docMap, SortedDocValues values, LongValues map) {
super(docMap);
this.values = values;
- this.maxDoc = maxDoc;
this.map = map;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
}
@@ -605,7 +469,23 @@ public abstract class DocValuesConsumer implements Closeable {
* The default implementation calls {@link #addSortedField}, passing
* an Iterable that merges ordinals and values and filters deleted documents .
*/
- public void mergeSortedField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedDocValues> toMerge) throws IOException {
+ public void mergeSortedField(FieldInfo fieldInfo, final MergeState mergeState) throws IOException {
+ List<SortedDocValues> toMerge = new ArrayList<>();
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.SORTED) {
+ values = docValuesProducer.getSorted(fieldInfo);
+ }
+ }
+ if (values == null) {
+ values = DocValues.emptySorted();
+ }
+ toMerge.add(values);
+ }
+
final int numReaders = toMerge.size();
final SortedDocValues dvs[] = toMerge.toArray(new SortedDocValues[numReaders]);
@@ -621,9 +501,10 @@ public abstract class DocValuesConsumer implements Closeable {
weights[sub] = dv.getValueCount();
} else {
LongBitSet bitset = new LongBitSet(dv.getValueCount());
- for (int i = 0; i < maxDoc; i++) {
- if (liveDocs.get(i)) {
- int ord = dv.getOrd(i);
+ int docID;
+ while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
+ if (liveDocs.get(docID)) {
+ int ord = dv.ordValue();
if (ord >= 0) {
bitset.set(ord);
}
@@ -639,122 +520,121 @@ public abstract class DocValuesConsumer implements Closeable {
// step 3: add field
addSortedField(fieldInfo,
- // ord -> value
- new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new Iterator<BytesRef>() {
- int currentOrd;
-
- @Override
- public boolean hasNext() {
- return currentOrd < map.getValueCount();
- }
-
- @Override
- public BytesRef next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- int segmentNumber = map.getFirstSegmentNumber(currentOrd);
- int segmentOrd = (int)map.getFirstSegmentOrd(currentOrd);
- final BytesRef term = dvs[segmentNumber].lookupOrd(segmentOrd);
- currentOrd++;
- return term;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- },
- // doc -> ord
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- // We must make a new DocIDMerger for each iterator:
- List<SortedDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new SortedDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i], map.getGlobalOrds(i)));
- }
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedDocValues getSorted(FieldInfo fieldInfoIn) {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong FieldInfo");
+ }
- final DocIDMerger<SortedDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
-
- return new Iterator<Number>() {
- int nextValue;
- boolean nextIsSet;
-
- @Override
- public boolean hasNext() {
- return nextIsSet || setNext();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- // TODO make a mutable number
- return nextValue;
- }
-
- private boolean setNext() {
- while (true) {
- SortedDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
-
- nextIsSet = true;
- int segOrd = sub.values.getOrd(sub.docID);
- nextValue = segOrd == -1 ? -1 : (int) sub.map.get(segOrd);
- return true;
- }
- }
- };
- }
- }
- );
+ // We must make new iterators + DocIDMerger for each iterator:
+
+ List<SortedDocValuesSub> subs = new ArrayList<>();
+ long cost = 0;
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.SORTED) {
+ try {
+ values = docValuesProducer.getSorted(readerFieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+ if (values == null) {
+ values = DocValues.emptySorted();
+ }
+ cost += values.cost();
+
+ subs.add(new SortedDocValuesSub(mergeState.docMaps[i], values, map.getGlobalOrds(i)));
+ }
+
+ final long finalCost = cost;
+
+ final DocIDMerger<SortedDocValuesSub> docIDMerger;
+ try {
+ docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new SortedDocValues() {
+ private int docID = -1;
+ private int ord;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ SortedDocValuesSub sub = docIDMerger.next();
+ if (sub == null) {
+ return NO_MORE_DOCS;
+ }
+ int subOrd = sub.values.ordValue();
+ assert subOrd != -1;
+ ord = (int) sub.map.get(subOrd);
+ docID = sub.mappedDocID;
+ return docID;
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ return finalCost;
+ }
+
+ @Override
+ public int getValueCount() {
+ return (int) map.getValueCount();
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ int segmentNumber = map.getFirstSegmentNumber(ord);
+ int segmentOrd = (int) map.getFirstSegmentOrd(ord);
+ return dvs[segmentNumber].lookupOrd(segmentOrd);
+ }
+ };
+ }
+ });
}
/** Tracks state of one sorted set sub-reader that we are merging */
private static class SortedSetDocValuesSub extends DocIDMerger.Sub {
- private final SortedSetDocValues values;
- int docID = -1;
- private final int maxDoc;
- private final LongValues map;
-
- public SortedSetDocValuesSub(MergeState.DocMap docMap, SortedSetDocValues values, int maxDoc, LongValues map) {
+ final SortedSetDocValues values;
+ final LongValues map;
+
+ public SortedSetDocValuesSub(MergeState.DocMap docMap, SortedSetDocValues values, LongValues map) {
super(docMap);
this.values = values;
- this.maxDoc = maxDoc;
this.map = map;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
@Override
public String toString() {
- return "SortedSetDocValuesSub(docID=" + docID + " mappedDocID=" + mappedDocID + " values=" + values + ")";
+ return "SortedSetDocValuesSub(mappedDocID=" + mappedDocID + " values=" + values + ")";
}
}
@@ -764,7 +644,23 @@ public abstract class DocValuesConsumer implements Closeable {
* The default implementation calls {@link #addSortedSetField}, passing
* an Iterable that merges ordinals and values and filters deleted documents .
*/
- public void mergeSortedSetField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedSetDocValues> toMerge) throws IOException {
+ public void mergeSortedSetField(FieldInfo mergeFieldInfo, final MergeState mergeState) throws IOException {
+
+ List<SortedSetDocValues> toMerge = new ArrayList<>();
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedSetDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_SET) {
+ values = docValuesProducer.getSortedSet(fieldInfo);
+ }
+ }
+ if (values == null) {
+ values = DocValues.emptySortedSet();
+ }
+ toMerge.add(values);
+ }
// step 1: iterate thru each sub and mark terms still in use
TermsEnum liveTerms[] = new TermsEnum[toMerge.size()];
@@ -778,9 +674,9 @@ public abstract class DocValuesConsumer implements Closeable {
weights[sub] = dv.getValueCount();
} else {
LongBitSet bitset = new LongBitSet(dv.getValueCount());
- for (int i = 0; i < maxDoc; i++) {
- if (liveDocs.get(i)) {
- dv.setDocument(i);
+ int docID;
+ while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
+ if (liveDocs.get(docID)) {
long ord;
while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
bitset.set(ord);
@@ -796,169 +692,102 @@ public abstract class DocValuesConsumer implements Closeable {
final OrdinalMap map = OrdinalMap.build(this, liveTerms, weights, PackedInts.COMPACT);
// step 3: add field
- addSortedSetField(fieldInfo,
- // ord -> value
- new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new Iterator<BytesRef>() {
- long currentOrd;
-
- @Override
- public boolean hasNext() {
- return currentOrd < map.getValueCount();
- }
-
- @Override
- public BytesRef next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- int segmentNumber = map.getFirstSegmentNumber(currentOrd);
- long segmentOrd = map.getFirstSegmentOrd(currentOrd);
- final BytesRef term = toMerge.get(segmentNumber).lookupOrd(segmentOrd);
- currentOrd++;
- return term;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- },
- // doc -> ord count
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
+ addSortedSetField(mergeFieldInfo,
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedSetDocValues getSortedSet(FieldInfo fieldInfo) {
+ if (fieldInfo != mergeFieldInfo) {
+ throw new IllegalArgumentException("wrong FieldInfo");
+ }
- // We must make a new DocIDMerger for each iterator:
- List<SortedSetDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new SortedSetDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i], map.getGlobalOrds(i)));
- }
+ // We must make new iterators + DocIDMerger for each iterator:
+ List<SortedSetDocValuesSub> subs = new ArrayList<>();
+
+ long cost = 0;
+
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedSetDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.SORTED_SET) {
+ try {
+ values = docValuesProducer.getSortedSet(readerFieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+ if (values == null) {
+ values = DocValues.emptySortedSet();
+ }
+ cost += values.cost();
+ subs.add(new SortedSetDocValuesSub(mergeState.docMaps[i], values, map.getGlobalOrds(i)));
+ }
+
+ final DocIDMerger<SortedSetDocValuesSub> docIDMerger;
+ try {
+ docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ final long finalCost = cost;
- final DocIDMerger<SortedSetDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
-
- return new Iterator<Number>() {
- int nextValue;
- boolean nextIsSet;
-
- @Override
- public boolean hasNext() {
- return nextIsSet || setNext();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- // TODO make a mutable number
- return nextValue;
- }
-
- private boolean setNext() {
- while (true) {
- SortedSetDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- sub.values.setDocument(sub.docID);
- nextValue = 0;
- while (sub.values.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
- nextValue++;
- }
- //System.out.println(" doc " + sub + " -> ord count = " + nextValue);
- nextIsSet = true;
- return true;
- }
- }
- };
- }
- },
- // ords
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
+ return new SortedSetDocValues() {
+ private int docID = -1;
+ private SortedSetDocValuesSub currentSub;
- // We must make a new DocIDMerger for each iterator:
- List<SortedSetDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new SortedSetDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i], map.getGlobalOrds(i)));
- }
+ @Override
+ public int docID() {
+ return docID;
+ }
- final DocIDMerger<SortedSetDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
-
- return new Iterator<Number>() {
- long nextValue;
- boolean nextIsSet;
- long ords[] = new long[8];
- int ordUpto;
- int ordLength;
-
- @Override
- public boolean hasNext() {
- return nextIsSet || setNext();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- if (hasNext() == false) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- // TODO make a mutable number
- return nextValue;
- }
-
- private boolean setNext() {
- while (true) {
- if (ordUpto < ordLength) {
- nextValue = ords[ordUpto];
- ordUpto++;
- nextIsSet = true;
- return true;
- }
-
- SortedSetDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- sub.values.setDocument(sub.docID);
-
- ordUpto = ordLength = 0;
- long ord;
- while ((ord = sub.values.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- if (ordLength == ords.length) {
- ords = ArrayUtil.grow(ords, ordLength+1);
- }
- ords[ordLength] = sub.map.get(ord);
- ordLength++;
- }
- continue;
- }
- }
- };
- }
- }
- );
+ @Override
+ public int nextDoc() throws IOException {
+ currentSub = docIDMerger.next();
+ if (currentSub == null) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = currentSub.mappedDocID;
+ }
+
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long nextOrd() throws IOException {
+ long subOrd = currentSub.values.nextOrd();
+ if (subOrd == NO_MORE_ORDS) {
+ return NO_MORE_ORDS;
+ }
+ return currentSub.map.get(subOrd);
+ }
+
+ @Override
+ public long cost() {
+ return finalCost;
+ }
+
+ @Override
+ public BytesRef lookupOrd(long ord) {
+ int segmentNumber = map.getFirstSegmentNumber(ord);
+ long segmentOrd = map.getFirstSegmentOrd(ord);
+ return toMerge.get(segmentNumber).lookupOrd(segmentOrd);
+ }
+
+ @Override
+ public long getValueCount() {
+ return map.getValueCount();
+ }
+ };
+ }
+ });
}
// TODO: seek-by-ord to nextSetBit
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
index b6e136b..9296f19 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
@@ -27,7 +27,6 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.Bits;
/** Abstract API that produces numeric, binary, sorted, sortedset,
* and sortednumeric docvalues.
@@ -65,12 +64,6 @@ public abstract class DocValuesProducer implements Closeable, Accountable {
* used by a single thread. */
public abstract SortedSetDocValues getSortedSet(FieldInfo field) throws IOException;
- /** Returns a {@link Bits} at the size of <code>reader.maxDoc()</code>,
- * with turned on bits for each docid that does have a value for this field.
- * The returned instance need not be thread-safe: it will only be
- * used by a single thread. */
- public abstract Bits getDocsWithField(FieldInfo field) throws IOException;
-
/**
* Checks consistency of this producer
* <p>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/LegacyDocValuesIterables.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/LegacyDocValuesIterables.java b/lucene/core/src/java/org/apache/lucene/codecs/LegacyDocValuesIterables.java
new file mode 100644
index 0000000..9d664c6
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/LegacyDocValuesIterables.java
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+
+import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+/** Bridge helper methods for legacy codecs to map sorted doc values to iterables. */
+
+public class LegacyDocValuesIterables {
+
+ private LegacyDocValuesIterables() {
+ // no
+ }
+
+ /** Converts {@link SortedDocValues} into an {@code Iterable<BytesRef>} for all the values.
+ *
+ * @deprecated Consume {@link SortedDocValues} instead. */
+ @Deprecated
+ public static Iterable<BytesRef> valuesIterable(final SortedDocValues values) {
+ return new Iterable<BytesRef>() {
+ @Override
+ public Iterator<BytesRef> iterator() {
+ return new Iterator<BytesRef>() {
+ private int nextOrd;
+
+ @Override
+ public boolean hasNext() {
+ return nextOrd < values.getValueCount();
+ }
+
+ @Override
+ public BytesRef next() {
+ return values.lookupOrd(nextOrd++);
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts {@link SortedSetDocValues} into an {@code Iterable<BytesRef>} for all the values.
+ *
+ * @deprecated Consume {@link SortedSetDocValues} instead. */
+ @Deprecated
+ public static Iterable<BytesRef> valuesIterable(final SortedSetDocValues values) {
+ return new Iterable<BytesRef>() {
+ @Override
+ public Iterator<BytesRef> iterator() {
+ return new Iterator<BytesRef>() {
+ private long nextOrd;
+
+ @Override
+ public boolean hasNext() {
+ return nextOrd < values.getValueCount();
+ }
+
+ @Override
+ public BytesRef next() {
+ return values.lookupOrd(nextOrd++);
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts {@link SortedDocValues} into the ord for each document as an {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link SortedDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> sortedOrdIterable(final DocValuesProducer valuesProducer, FieldInfo fieldInfo, int maxDoc) {
+ return new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+
+ final SortedDocValues values;
+ try {
+ values = valuesProducer.getSorted(fieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private int nextDocID;
+
+ @Override
+ public boolean hasNext() {
+ return nextDocID < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ try {
+ if (nextDocID > values.docID()) {
+ values.nextDoc();
+ }
+ int result;
+ if (nextDocID == values.docID()) {
+ result = values.ordValue();
+ } else {
+ result = -1;
+ }
+ nextDocID++;
+ return result;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts number-of-ords per document from {@link SortedSetDocValues} into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link SortedSetDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> sortedSetOrdCountIterable(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo, final int maxDoc) {
+
+ return new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+
+ final SortedSetDocValues values;
+ try {
+ values = valuesProducer.getSortedSet(fieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private int nextDocID;
+ private int ordCount;
+
+ @Override
+ public boolean hasNext() {
+ return nextDocID < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ try {
+ if (nextDocID > values.docID()) {
+ if (values.nextDoc() != NO_MORE_DOCS) {
+ ordCount = 0;
+ while (values.nextOrd() != NO_MORE_ORDS) {
+ ordCount++;
+ }
+ }
+ }
+ int result;
+ if (nextDocID == values.docID()) {
+ result = ordCount;
+ } else {
+ result = 0;
+ }
+ nextDocID++;
+ return result;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts all concatenated ords (in docID order) from {@link SortedSetDocValues} into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link SortedSetDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> sortedSetOrdsIterable(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo) {
+
+ return new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+
+ final SortedSetDocValues values;
+ try {
+ values = valuesProducer.getSortedSet(fieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private boolean nextIsSet;
+ private long nextOrd;
+
+ private void setNext() {
+ try {
+ if (nextIsSet == false) {
+ if (values.docID() == -1) {
+ values.nextDoc();
+ }
+ while (true) {
+ if (values.docID() == NO_MORE_DOCS) {
+ nextOrd = -1;
+ break;
+ }
+ nextOrd = values.nextOrd();
+ if (nextOrd != -1) {
+ break;
+ }
+ values.nextDoc();
+ }
+ nextIsSet = true;
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ setNext();
+ return nextOrd != -1;
+ }
+
+ @Override
+ public Number next() {
+ setNext();
+ assert nextOrd != -1;
+ nextIsSet = false;
+ return nextOrd;
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts number-of-values per document from {@link SortedNumericDocValues} into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link SortedDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> sortedNumericToDocCount(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo, int maxDoc) {
+ return new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+
+ final SortedNumericDocValues values;
+ try {
+ values = valuesProducer.getSortedNumeric(fieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private int nextDocID;
+ private int nextCount;
+
+ @Override
+ public boolean hasNext() {
+ return nextDocID < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ try {
+ if (nextDocID > values.docID()) {
+ values.nextDoc();
+ }
+ int result;
+ if (nextDocID == values.docID()) {
+ result = values.docValueCount();
+ } else {
+ result = 0;
+ }
+ nextDocID++;
+ return result;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts all concatenated values (in docID order) from {@link SortedNumericDocValues} into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link SortedDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> sortedNumericToValues(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo) {
+ return new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+
+ final SortedNumericDocValues values;
+ try {
+ values = valuesProducer.getSortedNumeric(fieldInfo);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private boolean nextIsSet;
+ private int nextCount;
+ private int upto;
+ private long nextValue;
+
+ private void setNext() {
+ try {
+ if (nextIsSet == false) {
+ if (upto == nextCount) {
+ values.nextDoc();
+ if (values.docID() == NO_MORE_DOCS) {
+ nextCount = 0;
+ nextIsSet = false;
+ return;
+ } else {
+ nextCount = values.docValueCount();
+ }
+ upto = 0;
+ }
+ nextValue = values.nextValue();
+ upto++;
+ nextIsSet = true;
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ setNext();
+ return nextCount != 0;
+ }
+
+ @Override
+ public Number next() {
+ setNext();
+ assert nextCount != 0;
+ nextIsSet = false;
+ return nextValue;
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts norms into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link NumericDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> normsIterable(final FieldInfo field, final NormsProducer normsProducer, final int maxDoc) {
+
+ return new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+
+ final NumericDocValues values;
+ try {
+ values = normsProducer.getNorms(field);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private int docIDUpto = -1;
+
+ @Override
+ public boolean hasNext() {
+ return docIDUpto+1 < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ docIDUpto++;
+ if (docIDUpto > values.docID()) {
+ try {
+ values.nextDoc();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ Number result;
+ if (docIDUpto == values.docID()) {
+ result = values.longValue();
+ } else {
+ // Unlike NumericDocValues, norms should return for missing values:
+ result = 0;
+ }
+ return result;
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts values from {@link BinaryDocValues} into {@code Iterable<BytesRef>}.
+ *
+ * @deprecated Consume {@link BinaryDocValues} instead. */
+ @Deprecated
+ public static Iterable<BytesRef> binaryIterable(final FieldInfo field, final DocValuesProducer valuesProducer, final int maxDoc) {
+ return new Iterable<BytesRef>() {
+ @Override
+ public Iterator<BytesRef> iterator() {
+
+ final BinaryDocValues values;
+ try {
+ values = valuesProducer.getBinary(field);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<BytesRef>() {
+ private int docIDUpto = -1;
+
+ @Override
+ public boolean hasNext() {
+ return docIDUpto+1 < maxDoc;
+ }
+
+ @Override
+ public BytesRef next() {
+ docIDUpto++;
+ if (docIDUpto > values.docID()) {
+ try {
+ values.nextDoc();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ BytesRef result;
+ if (docIDUpto == values.docID()) {
+ result = values.binaryValue();
+ } else {
+ result = null;
+ }
+ return result;
+ }
+ };
+ }
+ };
+ }
+
+ /** Converts values from {@link NumericDocValues} into {@code Iterable<Number>}.
+ *
+ * @deprecated Consume {@link NumericDocValues} instead. */
+ @Deprecated
+ public static Iterable<Number> numericIterable(final FieldInfo field, final DocValuesProducer valuesProducer, final int maxDoc) {
+ return new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+
+ final NumericDocValues values;
+ try {
+ values = valuesProducer.getNumeric(field);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ return new Iterator<Number>() {
+ private int docIDUpto = -1;
+
+ @Override
+ public boolean hasNext() {
+ return docIDUpto+1 < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ docIDUpto++;
+ if (docIDUpto > values.docID()) {
+ try {
+ values.nextDoc();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ Number result;
+ if (docIDUpto == values.docID()) {
+ result = values.longValue();
+ } else {
+ result = null;
+ }
+ return result;
+ }
+ };
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
index 40f44ab..ddd5c1b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
@@ -19,17 +19,13 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
-import java.util.NoSuchElementException;
import org.apache.lucene.index.DocIDMerger;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentWriteState;
-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/**
* Abstract API that consumes normalization values.
@@ -59,10 +55,10 @@ public abstract class NormsConsumer implements Closeable {
/**
* Writes normalization values for a field.
* @param field field information
- * @param values Iterable of numeric values (one for each document).
+ * @param normsProducer NormsProducer of the numeric norm values
* @throws IOException if an I/O error occurred.
*/
- public abstract void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException;
+ public abstract void addNormsField(FieldInfo field, NormsProducer normsProducer) throws IOException;
/** Merges in the fields from the readers in
* <code>mergeState</code>. The default implementation
@@ -78,22 +74,7 @@ public abstract class NormsConsumer implements Closeable {
}
for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
if (mergeFieldInfo.hasNorms()) {
- List<NumericDocValues> toMerge = new ArrayList<>();
- for (int i=0;i<mergeState.normsProducers.length;i++) {
- NormsProducer normsProducer = mergeState.normsProducers[i];
- NumericDocValues norms = null;
- if (normsProducer != null) {
- FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
- if (fieldInfo != null && fieldInfo.hasNorms()) {
- norms = normsProducer.getNorms(fieldInfo);
- }
- }
- if (norms == null) {
- norms = DocValues.emptyNumeric();
- }
- toMerge.add(norms);
- }
- mergeNormsField(mergeFieldInfo, mergeState, toMerge);
+ mergeNormsField(mergeFieldInfo, mergeState);
}
}
}
@@ -102,23 +83,16 @@ public abstract class NormsConsumer implements Closeable {
private static class NumericDocValuesSub extends DocIDMerger.Sub {
private final NumericDocValues values;
- private int docID = -1;
- private final int maxDoc;
-
- public NumericDocValuesSub(MergeState.DocMap docMap, NumericDocValues values, int maxDoc) {
+
+ public NumericDocValuesSub(MergeState.DocMap docMap, NumericDocValues values) {
super(docMap);
this.values = values;
- this.maxDoc = maxDoc;
+ assert values.docID() == -1;
}
@Override
- public int nextDoc() {
- docID++;
- if (docID == maxDoc) {
- return NO_MORE_DOCS;
- } else {
- return docID;
- }
+ public int nextDoc() throws IOException {
+ return values.nextDoc();
}
}
@@ -128,58 +102,85 @@ public abstract class NormsConsumer implements Closeable {
* The default implementation calls {@link #addNormsField}, passing
* an Iterable that merges and filters deleted documents on the fly.
*/
- public void mergeNormsField(final FieldInfo fieldInfo, final MergeState mergeState, final List<NumericDocValues> toMerge) throws IOException {
+ public void mergeNormsField(final FieldInfo mergeFieldInfo, final MergeState mergeState) throws IOException {
// TODO: try to share code with default merge of DVConsumer by passing MatchAllBits ?
- addNormsField(fieldInfo,
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
+ addNormsField(mergeFieldInfo,
+ new NormsProducer() {
+ @Override
+ public NumericDocValues getNorms(FieldInfo fieldInfo) throws IOException {
+ if (fieldInfo != mergeFieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
- // We must make a new DocIDMerger for each iterator:
List<NumericDocValuesSub> subs = new ArrayList<>();
- assert mergeState.docMaps.length == toMerge.size();
- for(int i=0;i<toMerge.size();i++) {
- subs.add(new NumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i]));
+ assert mergeState.docMaps.length == mergeState.docValuesProducers.length;
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ NumericDocValues norms = null;
+ NormsProducer normsProducer = mergeState.normsProducers[i];
+ if (normsProducer != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (readerFieldInfo != null && readerFieldInfo.hasNorms()) {
+ norms = normsProducer.getNorms(readerFieldInfo);
+ }
+ }
+
+ if (norms != null) {
+ subs.add(new NumericDocValuesSub(mergeState.docMaps[i], norms));
+ }
}
final DocIDMerger<NumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);
- return new Iterator<Number>() {
- long nextValue;
- boolean nextIsSet;
+ return new NumericDocValues() {
+ private int docID = -1;
+ private NumericDocValuesSub current;
@Override
- public boolean hasNext() {
- return nextIsSet || setNext();
+ public int docID() {
+ return docID;
}
@Override
- public void remove() {
+ public int nextDoc() throws IOException {
+ current = docIDMerger.next();
+ if (current == null) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = current.mappedDocID;
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- assert nextIsSet;
- nextIsSet = false;
- return nextValue;
+ public long cost() {
+ return 0;
}
- private boolean setNext() {
- NumericDocValuesSub sub = docIDMerger.next();
- if (sub == null) {
- return false;
- }
- nextIsSet = true;
- nextValue = sub.values.get(sub.docID);
- return true;
+ @Override
+ public long longValue() {
+ return current.values.longValue();
}
};
- }
- });
+ }
+
+ @Override
+ public void checkIntegrity() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+ });
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsConsumer.java
index b5f04eb..833500c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsConsumer.java
@@ -20,7 +20,9 @@ package org.apache.lucene.codecs.lucene53;
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
import org.apache.lucene.codecs.NormsConsumer;
+import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -55,7 +57,11 @@ class Lucene53NormsConsumer extends NormsConsumer {
}
@Override
- public void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
+ public void addNormsField(FieldInfo field, NormsProducer normsProducer) throws IOException {
+ addNormsField(field, LegacyDocValuesIterables.normsIterable(field, normsProducer, maxDoc));
+ }
+
+ private void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
meta.writeVInt(field.number);
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
[09/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index cf8372d..af3d391 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -513,7 +513,6 @@ public class TestPointQueries extends LuceneTestCase {
private void _run() throws Exception {
startingGun.await();
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for (int iter=0;iter<iters && failed.get() == false;iter++) {
Long lower = randomValue();
Long upper = randomValue();
@@ -578,8 +577,11 @@ public class TestPointQueries extends LuceneTestCase {
System.out.println(Thread.currentThread().getName() + ": hitCount: " + hits.cardinality());
}
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
for(int docID=0;docID<r.maxDoc();docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected = missing.get(id) == false && deleted.get(id) == false && values[id] >= lower && values[id] <= upper;
if (hits.get(docID) != expected) {
// We do exact quantized comparison so the bbox query should never disagree:
@@ -768,8 +770,6 @@ public class TestPointQueries extends LuceneTestCase {
private void _run() throws Exception {
startingGun.await();
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
for (int iter=0;iter<iters && failed.get() == false;iter++) {
byte[][] lower = new byte[numDims][];
@@ -837,9 +837,12 @@ public class TestPointQueries extends LuceneTestCase {
}
}
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
int failCount = 0;
for(int docID=0;docID<r.maxDoc();docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
if (hits.get(docID) != expected.get(id)) {
System.out.println("FAIL: iter=" + iter + " id=" + id + " docID=" + docID + " expected=" + expected.get(id) + " but got " + hits.get(docID) + " deleted?=" + deleted.get(id) + " missing?=" + missing.get(id));
for(int dim=0;dim<numDims;dim++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index 83144b1..de96755 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -77,7 +77,9 @@ public class TestSimilarityProvider extends LuceneTestCase {
NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertFalse(fooNorms.get(i) == barNorms.get(i));
+ assertEquals(i, fooNorms.nextDoc());
+ assertEquals(i, barNorms.nextDoc());
+ assertFalse(fooNorms.longValue() == barNorms.longValue());
}
// sanity check of searching
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
index 2bccaff..5c06a74 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
@@ -239,10 +239,11 @@ public class TestSortRandom extends LuceneTestCase {
assertNotNull(idSource);
final FixedBitSet bits = new FixedBitSet(maxDoc);
for(int docID=0;docID<maxDoc;docID++) {
+ assertEquals(docID, idSource.nextDoc());
if (random.nextFloat() <= density) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
- matchValues.add(docValues.get((int) idSource.get(docID)));
+ matchValues.add(docValues.get((int) idSource.longValue()));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
index 774a2ce..4287125 100644
--- a/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
+++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
@@ -19,7 +19,7 @@ package org.apache.lucene.util.packed;
import java.util.Random;
-import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.LegacyNumericDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -46,7 +46,7 @@ public class TestDirectPacked extends LuceneTestCase {
writer.finish();
output.close();
IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
- NumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
+ LegacyNumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
assertEquals(1, reader.get(0));
assertEquals(0, reader.get(1));
assertEquals(2, reader.get(2));
@@ -110,7 +110,7 @@ public class TestDirectPacked extends LuceneTestCase {
writer.finish();
output.close();
IndexInput input = directory.openInput(name, IOContext.DEFAULT);
- NumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsRequired, offset);
+ LegacyNumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsRequired, offset);
for (int j = 0; j < original.length; j++) {
assertEquals("bpv=" + bpv, original[j], reader.get(j));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java b/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
index e64249e..1c11cb2 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
@@ -36,7 +36,7 @@ import org.apache.lucene.search.SortField;
* SimpleBindings bindings = new SimpleBindings();
* // document's text relevance score
* bindings.add(new SortField("_score", SortField.Type.SCORE));
- * // integer NumericDocValues field (or from FieldCache)
+ * // integer NumericDocValues field
* bindings.add(new SortField("popularity", SortField.Type.INT));
* // another expression
* bindings.add("recency", myRecencyExpression);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
index c78f6a97..13174bc 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
@@ -154,7 +154,7 @@ public final class JavascriptCompiler {
* use the FunctionValues class.
*/
@SuppressWarnings({"unused", "null"})
- private static void unusedTestCompile() {
+ private static void unusedTestCompile() throws IOException {
FunctionValues f = null;
double ret = f.doubleVal(2);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
index 3129d8c..eeb3c9c 100644
--- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
+++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
@@ -136,6 +136,7 @@ public class TestExpressionValueSource extends LuceneTestCase {
assertEquals(DocIdSetIterator.NO_MORE_DOCS, iter.nextDoc());
// just the first doc
+ values = vs.getValues(new HashMap<String,Object>(), leaf);
scorer = values.getRangeScorer(leaf, "4", "40", false, false);
iter = scorer.iterator();
assertEquals(-1, scorer.docID());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
index 936c962..7bbe94a 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
@@ -113,8 +113,8 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
if (map == null) {
// uncached, or not a multi dv
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(origReader, field);
- if (dv instanceof MultiSortedSetDocValues) {
- map = ((MultiSortedSetDocValues)dv).mapping;
+ if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
+ map = ((MultiDocValues.MultiSortedSetDocValues)dv).mapping;
if (map.owner == origReader.getCoreCacheKey()) {
cachedOrdMaps.put(field, map);
}
@@ -127,6 +127,7 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
int size = origReader.leaves().size();
final SortedSetDocValues[] values = new SortedSetDocValues[size];
final int[] starts = new int[size+1];
+ long cost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = origReader.leaves().get(i);
final LeafReader reader = context.reader();
@@ -140,9 +141,10 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
}
values[i] = v;
starts[i] = context.docBase;
+ cost += v.cost();
}
starts[size] = origReader.maxDoc();
- return new MultiSortedSetDocValues(values, starts, map);
+ return new MultiSortedSetDocValues(values, starts, map, cost);
}
/** Returns mapping from prefix to {@link OrdRange}. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
index a571328..38fb174 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
@@ -25,16 +25,16 @@ import java.util.Map;
import org.apache.lucene.facet.FacetResult;
import org.apache.lucene.facet.Facets;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.TopOrdAndIntQueue;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState.OrdRange;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -151,7 +151,7 @@ public class SortedSetDocValuesFacetCounts extends Facets {
// TODO: is this right? really, we need a way to
// verify that this ordinalMap "matches" the leaves in
// matchingDocs...
- if (dv instanceof MultiSortedSetDocValues && matchingDocs.size() > 1) {
+ if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
} else {
ordinalMap = null;
@@ -199,12 +199,16 @@ public class SortedSetDocValuesFacetCounts extends Facets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- //System.out.println(" segOrd=" + segOrd + " ord=" + term + " globalOrd=" + ordinalMap.getGlobalOrd(segOrd, term));
- counts[(int) ordMap.get(term)]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ //System.out.println(" segOrd=" + segOrd + " ord=" + term + " globalOrd=" + ordinalMap.getGlobalOrd(segOrd, term));
+ counts[(int) ordMap.get(term)]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
} else {
@@ -215,12 +219,16 @@ public class SortedSetDocValuesFacetCounts extends Facets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- //System.out.println(" ord=" + term);
- segCounts[term]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ //System.out.println(" ord=" + term);
+ segCounts[term]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
@@ -238,11 +246,15 @@ public class SortedSetDocValuesFacetCounts extends Facets {
// just aggregate directly into counts:
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- counts[term]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ counts[term]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
index 6afe5c7..fd48849 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
@@ -19,9 +19,9 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -50,9 +50,24 @@ public class DocValuesOrdinalsReader extends OrdinalsReader {
final BinaryDocValues values = values0;
return new OrdinalsSegmentReader() {
+
+ private int lastDocID;
+
@Override
public void get(int docID, IntsRef ordinals) throws IOException {
- final BytesRef bytes = values.get(docID);
+ if (docID < lastDocID) {
+ throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ final BytesRef bytes;
+ if (values.docID() == docID) {
+ bytes = values.binaryValue();
+ } else {
+ bytes = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
decode(bytes, ordinals);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
index 0ed8f22..7ad5430 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -59,20 +59,25 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int ord = 0;
- int offset = bytesRef.offset;
- int prev = 0;
- while (offset < end) {
- byte b = bytes[offset++];
- if (b >= 0) {
- prev = ord = ((ord << 7) | b) + prev;
- ++values[ord];
- ord = 0;
- } else {
- ord = (ord << 7) | (b & 0x7F);
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int ord = 0;
+ int offset = bytesRef.offset;
+ int prev = 0;
+ while (offset < end) {
+ byte b = bytes[offset++];
+ if (b >= 0) {
+ prev = ord = ((ord << 7) | b) + prev;
+ ++values[ord];
+ ord = 0;
+ } else {
+ ord = (ord << 7) | (b & 0x7F);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
index f116cd9..cb798af 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
@@ -20,13 +20,14 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.FacetsConfig.DimConfig;
+import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.taxonomy.OrdinalsReader.OrdinalsSegmentReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FilterBinaryDocValues;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -75,25 +76,26 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
}
- private class OrdinalMappingBinaryDocValues extends BinaryDocValues {
+ private class OrdinalMappingBinaryDocValues extends FilterBinaryDocValues {
private final IntsRef ordinals = new IntsRef(32);
private final OrdinalsSegmentReader ordsReader;
- OrdinalMappingBinaryDocValues(OrdinalsSegmentReader ordsReader) throws IOException {
+ OrdinalMappingBinaryDocValues(OrdinalsSegmentReader ordsReader, BinaryDocValues in) throws IOException {
+ super(in);
this.ordsReader = ordsReader;
}
@SuppressWarnings("synthetic-access")
@Override
- public BytesRef get(int docID) {
+ public BytesRef binaryValue() {
try {
// NOTE: this isn't quite koscher, because in general
// multiple threads can call BinaryDV.get which would
// then conflict on the single ordinals instance, but
// because this impl is only used for merging, we know
// only 1 thread calls us:
- ordsReader.get(docID, ordinals);
+ ordsReader.get(docID(), ordinals);
// map the ordinals
for (int i = 0; i < ordinals.length; i++) {
@@ -102,7 +104,7 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
return encode(ordinals);
} catch (IOException e) {
- throw new RuntimeException("error reading category ordinals for doc " + docID, e);
+ throw new RuntimeException("error reading category ordinals for doc " + docID(), e);
}
}
}
@@ -150,7 +152,7 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
if (facetFields.contains(field)) {
final OrdinalsReader ordsReader = getOrdinalsReader(field);
- return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext()));
+ return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext()), in.getBinaryDocValues(field));
} else {
return in.getBinaryDocValues(field);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
index 7dd31d3..ce82e27 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
index a76e750..5a71aa2 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -61,22 +61,27 @@ public class TaxonomyFacetSumFloatAssociations extends FloatTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int offset = bytesRef.offset;
- while (offset < end) {
- int ord = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- int value = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- values[ord] += Float.intBitsToFloat(value);
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int offset = bytesRef.offset;
+ while (offset < end) {
+ int ord = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ int value = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ values[ord] += Float.intBitsToFloat(value);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
index 64001ad..6913d0d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -61,22 +61,27 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int offset = bytesRef.offset;
- while (offset < end) {
- int ord = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- int value = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- values[ord] += value;
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int offset = bytesRef.offset;
+ while (offset < end) {
+ int ord = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ int value = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ values[ord] += value;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java b/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
index ea29c68..1a7633b 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
@@ -296,4 +296,4 @@ public class TestMultipleIndexFields extends FacetTestCase {
iw.addDocument(config.build(tw, doc));
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
index 0978b50..db3c59e 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
@@ -28,8 +28,8 @@ import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
@@ -106,7 +106,9 @@ public class TestOrdinalMappingLeafReader extends FacetTestCase {
BinaryDocValues bdv = MultiDocValues.getBinaryValues(indexReader, "bdv");
BinaryDocValues cbdv = MultiDocValues.getBinaryValues(indexReader, "cbdv");
for (int i = 0; i < indexReader.maxDoc(); i++) {
- assertEquals(Integer.parseInt(cbdv.get(i).utf8ToString()), Integer.parseInt(bdv.get(i).utf8ToString())*2);
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(i, cbdv.nextDoc());
+ assertEquals(Integer.parseInt(cbdv.binaryValue().utf8ToString()), Integer.parseInt(bdv.binaryValue().utf8ToString())*2);
}
IOUtils.close(indexReader, taxoReader);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
index 4c386b6..312a3e2 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
@@ -338,7 +338,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
* @param doc The specified doc
* @return the group value for the specified doc
*/
- protected abstract GROUP_VALUE_TYPE getDocGroupValue(int doc);
+ protected abstract GROUP_VALUE_TYPE getDocGroupValue(int doc) throws IOException;
/**
* Returns a copy of the specified group value by creating a new instance and copying the value from the specified
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
index 36579a6..0237e21 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
@@ -61,7 +61,7 @@ public class FunctionFirstPassGroupingCollector extends AbstractFirstPassGroupin
}
@Override
- protected MutableValue getDocGroupValue(int doc) {
+ protected MutableValue getDocGroupValue(int doc) throws IOException {
filler.fillValue(doc);
return mval;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
index ae01e33..14636ec 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
@@ -16,8 +16,15 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
@@ -29,17 +36,10 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* A base implementation of {@link org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector} for retrieving the most relevant groups when grouping
* on a string based group field. More specifically this all concrete implementations of this base implementation
- * use {@link org.apache.lucene.index.SortedDocValues}.
+ * use {@link SortedDocValues}.
*
* @lucene.experimental
*/
@@ -131,15 +131,30 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ protected int getOrdForDoc(int doc) throws IOException {
+ int curDocID = groupIndex.docID();
+ if (curDocID < doc) {
+ curDocID = groupIndex.advance(doc);
+ }
+
+ if (curDocID == doc) {
+ return groupIndex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- final int ord = groupIndex.getOrd(doc);
+ int ord = getOrdForDoc(doc);
+
BytesRef groupValue;
if (ord == -1) {
groupValue = null;
} else {
groupValue = groupIndex.lookupOrd(ord);
}
+
GroupHead groupHead = groups.get(groupValue);
if (groupHead == null) {
groupValue = groupValue == null ? null : BytesRef.deepCopyOf(groupValue);
@@ -263,9 +278,22 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
this.scorer = scorer;
}
+ private int getOrdForDoc(int doc) throws IOException {
+ int curDocID = groupIndex.docID();
+ if (curDocID < doc) {
+ curDocID = groupIndex.advance(doc);
+ }
+
+ if (curDocID == doc) {
+ return groupIndex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ int key = getOrdForDoc(doc);
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
@@ -328,6 +356,15 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ void setDoc(int docID) throws IOException {
+ for (int i = 0; i < sortsIndex.length; i++) {
+ SortedDocValues values = sortsIndex[i];
+ if (values != null && docID > values.docID()) {
+ values.advance(docID);
+ }
+ }
+ }
+
class GroupHead extends AbstractAllGroupHeadsCollector.GroupHead<BytesRef> {
BytesRefBuilder[] sortValues;
@@ -339,14 +376,19 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
sortValues = new BytesRefBuilder[sortsIndex.length];
sortOrds = new int[sortsIndex.length];
scores = new float[sortsIndex.length];
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
if (fields[i].getType() == SortField.Type.SCORE) {
scores[i] = scorer.score();
} else {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
+ if (doc == sortsIndex[i].docID()) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ } else {
+ sortOrds[i] = -1;
+ }
sortValues[i] = new BytesRefBuilder();
if (sortOrds[i] != -1) {
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
}
}
}
@@ -363,24 +405,44 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
return 0;
} else {
+ if (sortsIndex[compIDX].docID() < doc) {
+ sortsIndex[compIDX].advance(doc);
+ }
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- final BytesRef term = sortsIndex[compIDX].get(doc);
+ final BytesRef term;
+ if (sortsIndex[compIDX].docID() == doc) {
+ term = sortsIndex[compIDX].binaryValue();
+ } else {
+ term = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
return sortValues[compIDX].get().compareTo(term);
} else {
- return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
+ int ord;
+ if (sortsIndex[compIDX].docID() == doc) {
+ ord = sortsIndex[compIDX].ordValue();
+ } else {
+ ord = -1;
+ }
+ return sortOrds[compIDX] - ord;
}
}
}
@Override
public void updateDocHead(int doc) throws IOException {
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
if (fields[i].getType() == SortField.Type.SCORE) {
scores[i] = scorer.score();
} else {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortsIndex[i].docID() == doc) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ } else {
+ sortOrds[i] = -1;
+ sortValues[i].clear();
+ }
}
}
this.doc = doc + readerContext.docBase;
@@ -429,7 +491,17 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ if (doc > groupIndex.docID()) {
+ groupIndex.advance(doc);
+ }
+
+ int key;
+ if (doc == groupIndex.docID()) {
+ key = groupIndex.ordValue();
+ } else {
+ key = -1;
+ }
+
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
@@ -485,38 +557,74 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ void setDoc(int docID) throws IOException {
+ for (int i = 0; i < sortsIndex.length; i++) {
+ SortedDocValues values = sortsIndex[i];
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ }
+ }
+
class GroupHead extends AbstractAllGroupHeadsCollector.GroupHead<BytesRef> {
BytesRefBuilder[] sortValues;
int[] sortOrds;
- GroupHead(int doc, BytesRef groupValue) {
+ GroupHead(int doc, BytesRef groupValue) throws IOException {
super(groupValue, doc + readerContext.docBase);
sortValues = new BytesRefBuilder[sortsIndex.length];
sortOrds = new int[sortsIndex.length];
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
+ if (doc == sortsIndex[i].docID()) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ } else {
+ sortOrds[i] = -1;
+ }
sortValues[i] = new BytesRefBuilder();
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortOrds[i] != -1) {
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ }
}
}
@Override
public int compare(int compIDX, int doc) throws IOException {
+ if (sortsIndex[compIDX].docID() < doc) {
+ sortsIndex[compIDX].advance(doc);
+ }
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- final BytesRef term = sortsIndex[compIDX].get(doc);
+ final BytesRef term;
+ if (sortsIndex[compIDX].docID() == doc) {
+ term = sortsIndex[compIDX].binaryValue();
+ } else {
+ term = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
return sortValues[compIDX].get().compareTo(term);
} else {
- return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
+ int ord;
+ if (sortsIndex[compIDX].docID() == doc) {
+ ord = sortsIndex[compIDX].ordValue();
+ } else {
+ ord = -1;
+ }
+ return sortOrds[compIDX] - ord;
}
}
@Override
public void updateDocHead(int doc) throws IOException {
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortsIndex[i].docID() == doc) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ } else {
+ sortOrds[i] = -1;
+ sortValues[i].clear();
+ }
}
this.doc = doc + readerContext.docBase;
}
@@ -566,7 +674,17 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ if (doc > groupIndex.docID()) {
+ groupIndex.advance(doc);
+ }
+
+ int key;
+ if (doc == groupIndex.docID()) {
+ key = groupIndex.ordValue();
+ } else {
+ key = -1;
+ }
+
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
index f79964d..0009c36 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
@@ -16,18 +16,18 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
/**
* A collector that collects all groups that match the
* query. Only the group value is collected, and the order
@@ -81,7 +81,15 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
@Override
public void collect(int doc) throws IOException {
- int key = index.getOrd(doc);
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+ int key;
+ if (doc == index.docID()) {
+ key = index.ordValue();
+ } else {
+ key = -1;
+ }
if (!ordSet.exists(key)) {
ordSet.put(key);
final BytesRef term;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
index ab492d0..7980c48 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
@@ -16,20 +16,20 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.grouping.AbstractDistinctValuesCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
/**
* A term based implementation of {@link org.apache.lucene.search.grouping.AbstractDistinctValuesCollector} that relies
* on {@link SortedDocValues} to count the distinct values per group.
@@ -67,13 +67,32 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
@Override
public void collect(int doc) throws IOException {
- int slot = ordSet.find(groupFieldTermIndex.getOrd(doc));
+ if (doc > groupFieldTermIndex.docID()) {
+ groupFieldTermIndex.advance(doc);
+ }
+ int ord;
+ if (doc == groupFieldTermIndex.docID()) {
+ ord = groupFieldTermIndex.ordValue();
+ } else {
+ ord = -1;
+ }
+ int slot = ordSet.find(ord);
if (slot < 0) {
return;
}
GroupCount gc = groupCounts[slot];
- int countOrd = countFieldTermIndex.getOrd(doc);
+ if (doc > countFieldTermIndex.docID()) {
+ countFieldTermIndex.advance(doc);
+ }
+
+ int countOrd;
+ if (doc == countFieldTermIndex.docID()) {
+ countOrd = countFieldTermIndex.ordValue();
+ } else {
+ countOrd = -1;
+ }
+
if (doesNotContainOrd(countOrd, gc.ords)) {
if (countOrd == -1) {
gc.uniqueValues.add(null);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
index 4c50254..063521f 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
@@ -18,8 +18,8 @@ package org.apache.lucene.search.grouping.term;
import java.io.IOException;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector;
@@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector} that groups based on
- * field values and more specifically uses {@link org.apache.lucene.index.SortedDocValues}
+ * field values and more specifically uses {@link SortedDocValues}
* to collect groups.
*
* @lucene.experimental
@@ -61,12 +61,14 @@ public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCol
}
@Override
- protected BytesRef getDocGroupValue(int doc) {
- final int ord = index.getOrd(doc);
- if (ord == -1) {
- return null;
+ protected BytesRef getDocGroupValue(int doc) throws IOException {
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+ if (doc == index.docID()) {
+ return index.binaryValue();
} else {
- return index.lookupOrd(ord);
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
index 5c36fe1..a48df03 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
@@ -16,8 +16,12 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
@@ -27,10 +31,6 @@ import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.UnicodeUtil;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* An implementation of {@link AbstractGroupFacetCollector} that computes grouped facets based on the indexed terms
* from DocValues.
@@ -86,12 +86,31 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
@Override
public void collect(int doc) throws IOException {
- int facetOrd = facetFieldTermsIndex.getOrd(doc);
+ if (doc > facetFieldTermsIndex.docID()) {
+ facetFieldTermsIndex.advance(doc);
+ }
+
+ int facetOrd;
+ if (doc == facetFieldTermsIndex.docID()) {
+ facetOrd = facetFieldTermsIndex.ordValue();
+ } else {
+ facetOrd = -1;
+ }
+
if (facetOrd < startFacetOrd || facetOrd >= endFacetOrd) {
return;
}
- int groupOrd = groupFieldTermsIndex.getOrd(doc);
+ if (doc > groupFieldTermsIndex.docID()) {
+ groupFieldTermsIndex.advance(doc);
+ }
+
+ int groupOrd;
+ if (doc == groupFieldTermsIndex.docID()) {
+ groupOrd = groupFieldTermsIndex.ordValue();
+ } else {
+ groupOrd = -1;
+ }
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.getValueCount()+1) + facetOrd;
if (segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
return;
@@ -206,7 +225,17 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
@Override
public void collect(int doc) throws IOException {
- int groupOrd = groupFieldTermsIndex.getOrd(doc);
+ if (doc > groupFieldTermsIndex.docID()) {
+ groupFieldTermsIndex.advance(doc);
+ }
+
+ int groupOrd;
+ if (doc == groupFieldTermsIndex.docID()) {
+ groupOrd = groupFieldTermsIndex.ordValue();
+ } else {
+ groupOrd = -1;
+ }
+
if (facetFieldNumTerms == 0) {
int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1);
if (facetPrefix != null || segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
@@ -227,12 +256,16 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
return;
}
- facetFieldDocTermOrds.setDocument(doc);
- long ord;
+ if (doc > facetFieldDocTermOrds.docID()) {
+ facetFieldDocTermOrds.advance(doc);
+ }
boolean empty = true;
- while ((ord = facetFieldDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- process(groupOrd, (int) ord);
- empty = false;
+ if (doc == facetFieldDocTermOrds.docID()) {
+ long ord;
+ while ((ord = facetFieldDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ process(groupOrd, (int) ord);
+ empty = false;
+ }
}
if (empty) {
@@ -379,4 +412,4 @@ class GroupedFacetHit {
this.groupValue = groupValue;
this.facetValue = facetValue;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
index 9292856..c7b16e9 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
@@ -19,8 +19,8 @@ package org.apache.lucene.search.grouping.term;
import java.io.IOException;
import java.util.Collection;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
@@ -30,7 +30,7 @@ import org.apache.lucene.util.SentinelIntSet;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector} that groups based on
- * field values and more specifically uses {@link org.apache.lucene.index.SortedDocValues}
+ * field values and more specifically uses {@link SortedDocValues}
* to collect grouped docs.
*
* @lucene.experimental
@@ -70,7 +70,18 @@ public class TermSecondPassGroupingCollector extends AbstractSecondPassGroupingC
@Override
protected SearchGroupDocs<BytesRef> retrieveGroup(int doc) throws IOException {
- int slot = ordSet.find(index.getOrd(doc));
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+
+ int ord;
+ if (doc == index.docID()) {
+ ord = index.ordValue();
+ } else {
+ ord = -1;
+ }
+
+ int slot = ordSet.find(ord);
if (slot >= 0) {
return groupDocs[slot];
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
index 9bdb8f5..316676b 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
@@ -283,10 +283,13 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
final DirectoryReader r = w.getReader();
w.close();
- final NumericDocValues docIdToFieldId = MultiDocValues.getNumericValues(r, "id");
+ NumericDocValues values = MultiDocValues.getNumericValues(r, "id");
+ final int[] docIDToFieldId = new int[numDocs];
final int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < numDocs; i++) {
- int fieldId = (int) docIdToFieldId.get(i);
+ assertEquals(i, values.nextDoc());
+ int fieldId = (int) values.longValue();
+ docIDToFieldId[i] = fieldId;
fieldIdToDocID[fieldId] = i;
}
@@ -296,7 +299,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
for (int contentID = 0; contentID < 3; contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
for (ScoreDoc hit : hits) {
- int idValue = (int) docIdToFieldId.get(hit.doc);
+ int idValue = docIDToFieldId[hit.doc];
final GroupDoc gd = groupDocs[idValue];
assertEquals(gd.id, idValue);
seenIDs.add(idValue);
@@ -329,7 +332,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.length; i++) {
- actualGroupHeads[i] = (int) docIdToFieldId.get(actualGroupHeads[i]);
+ actualGroupHeads[i] = docIDToFieldId[actualGroupHeads[i]];
}
// Allows us the easily iterate and assert the actual and expected results.
Arrays.sort(expectedGroupHeads);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
index 2f51c94..4a59632 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
@@ -757,8 +757,13 @@ public class TestGrouping extends LuceneTestCase {
final DirectoryReader r = w.getReader();
w.close();
-
- final NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
+ NumericDocValues values = MultiDocValues.getNumericValues(r, "id");
+ int[] docIDToID = new int[r.maxDoc()];
+ for(int i=0;i<r.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToID[i] = (int) values.longValue();
+ }
DirectoryReader rBlocks = null;
Directory dirBlocks = null;
@@ -773,7 +778,7 @@ public class TestGrouping extends LuceneTestCase {
for(int contentID=0;contentID<3;contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
- int idValue = (int) docIDToID.get(hit.doc);
+ int idValue = docIDToID[hit.doc];
final GroupDoc gd = groupDocs[idValue];
seenIDs.add(idValue);
@@ -796,8 +801,6 @@ public class TestGrouping extends LuceneTestCase {
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Query lastDocInBlock = new TermQuery(new Term("groupend", "x"));
- final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
- assertNotNull(docIDToIDBlocks);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);
@@ -806,6 +809,14 @@ public class TestGrouping extends LuceneTestCase {
// means a monotonic shift in scores, so we can
// reliably remap them w/ Map:
final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
+
+ values = MultiDocValues.getNumericValues(rBlocks, "id");
+ assertNotNull(values);
+ int[] docIDToIDBlocks = new int[rBlocks.maxDoc()];
+ for(int i=0;i<rBlocks.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToIDBlocks[i] = (int) values.longValue();
+ }
// Tricky: must separately set .score2, because the doc
// block index was created with possible deletions!
@@ -818,11 +829,11 @@ public class TestGrouping extends LuceneTestCase {
//" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
- final GroupDoc gd = groupDocsByID[(int) docIDToIDBlocks.get(hit.doc)];
+ final GroupDoc gd = groupDocsByID[docIDToIDBlocks[hit.doc]];
assertTrue(gd.score2 == 0.0);
gd.score2 = hit.score;
- assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
- //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
+ assertEquals(gd.id, docIDToIDBlocks[hit.doc]);
+ //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks[hit.doc]);
termScoreMap.put(gd.score, gd.score2);
}
}
@@ -1005,13 +1016,13 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : groupsResult.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToID[sd.doc] + " score=" + sd.score);
}
}
if (searchIter == 14) {
for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
- System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
+ System.out.println("ID=" + docIDToID[docIDX] + " explain=" + s.explain(query, docIDX));
}
}
}
@@ -1023,7 +1034,7 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToID[sd.doc] + " score=" + sd.score);
}
}
}
@@ -1072,7 +1083,7 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToIDBlocks[sd.doc] + " score=" + sd.score);
if (first) {
System.out.println("explain: " + sBlocks.explain(query, sd.doc));
first = false;
@@ -1237,7 +1248,7 @@ public class TestGrouping extends LuceneTestCase {
}
}
- private void assertEquals(NumericDocValues docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
+ private void assertEquals(int[] docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
if (expected == null) {
assertNull(actual);
return;
@@ -1284,8 +1295,8 @@ public class TestGrouping extends LuceneTestCase {
for(int docIDX=0;docIDX<expectedFDs.length;docIDX++) {
final FieldDoc expectedFD = (FieldDoc) expectedFDs[docIDX];
final FieldDoc actualFD = (FieldDoc) actualFDs[docIDX];
- //System.out.println(" actual doc=" + docIDtoID.get(actualFD.doc) + " score=" + actualFD.score);
- assertEquals(expectedFD.doc, docIDtoID.get(actualFD.doc));
+ //System.out.println(" actual doc=" + docIDtoID[actualFD.doc] + " score=" + actualFD.score);
+ assertEquals(expectedFD.doc, docIDtoID[actualFD.doc]);
if (testScores) {
assertEquals(expectedFD.score, actualFD.score, 0.1);
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 55f360a..07506f2 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -129,11 +129,6 @@ public class TermVectorLeafReader extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- return null;
- }
-
- @Override
public NumericDocValues getNormValues(String field) throws IOException {
return null;//Is this needed? See MemoryIndex for a way to do it.
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
index 7f79809..1b277f1 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
@@ -63,9 +63,9 @@ import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
@@ -473,11 +473,6 @@ public class WeightedSpanTermExtractor {
public NumericDocValues getNormValues(String field) throws IOException {
return super.getNormValues(FIELD_NAME);
}
-
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- return super.getDocsWithField(FIELD_NAME);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java b/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
index 507d6a6..e034369 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
@@ -16,14 +16,14 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
-import java.io.IOException;
-
abstract class BaseGlobalOrdinalScorer extends Scorer {
final SortedDocValues values;
@@ -64,4 +64,4 @@ abstract class BaseGlobalOrdinalScorer extends Scorer {
protected abstract TwoPhaseIterator createTwoPhaseIterator(DocIdSetIterator approximation);
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java b/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
index 509c1e9..212d75b 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
@@ -16,7 +16,8 @@
*/
package org.apache.lucene.search.join;
-import org.apache.lucene.index.DocValues;
+import java.io.IOException;
+
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
@@ -96,37 +97,77 @@ public class BlockJoinSelector {
* one value per parent among its {@code children} using the configured
* {@code selection} type. */
public static SortedDocValues wrap(final SortedDocValues values, Type selection, BitSet parents, BitSet children) {
+ if (values.docID() != -1) {
+ throw new IllegalArgumentException("values iterator was already consumed: values.docID=" + values.docID());
+ }
return new SortedDocValues() {
+ private int ord;
+ private int docID = -1;
+
@Override
- public int getOrd(int docID) {
- assert parents.get(docID) : "this selector may only be used on parent documents";
+ public int docID() {
+ return docID;
+ }
- if (docID == 0) {
- // no children
- return -1;
+ @Override
+ public int nextDoc() throws IOException {
+ assert docID != NO_MORE_DOCS;
+
+ if (values.docID() == -1) {
+ if (values.nextDoc() == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
}
- final int firstChild = parents.prevSetBit(docID - 1) + 1;
+ if (values.docID() == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+
+ int nextParentDocID = parents.nextSetBit(values.docID());
+ ord = values.ordValue();
- int ord = -1;
- for (int child = children.nextSetBit(firstChild); child < docID; child = children.nextSetBit(child + 1)) {
- final int childOrd = values.getOrd(child);
- switch (selection) {
- case MIN:
- if (ord == -1) {
- ord = childOrd;
- } else if (childOrd != -1) {
- ord = Math.min(ord, childOrd);
- }
- break;
- case MAX:
- ord = Math.max(ord, childOrd);
- break;
- default:
- throw new AssertionError();
+ while (true) {
+ int childDocID = values.nextDoc();
+ assert childDocID != nextParentDocID;
+ if (childDocID > nextParentDocID) {
+ break;
+ }
+ if (children.get(childDocID) == false) {
+ continue;
}
+ if (selection == Type.MIN) {
+ ord = Math.min(ord, values.ordValue());
+ } else if (selection == Type.MAX) {
+ ord = Math.max(ord, values.ordValue());
+ } else {
+ throw new AssertionError();
+ }
+ }
+
+ docID = nextParentDocID;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= parents.length()) {
+ docID = NO_MORE_DOCS;
+ return docID;
}
+ if (target == 0) {
+ assert docID() == -1;
+ return nextDoc();
+ }
+ int prevParentDocID = parents.prevSetBit(target-1);
+ values.advance(prevParentDocID+1);
+ return nextDoc();
+ }
+
+ @Override
+ public int ordValue() {
return ord;
}
@@ -140,6 +181,10 @@ public class BlockJoinSelector {
return values.getValueCount();
}
+ @Override
+ public long cost() {
+ return values.cost();
+ }
};
}
@@ -158,53 +203,102 @@ public class BlockJoinSelector {
default:
throw new AssertionError();
}
- return wrap(values, DocValues.docsWithValue(sortedNumerics, parents.length()), selection, parents, children);
+ return wrap(values, selection, parents, children);
}
- /** Wraps the provided {@link NumericDocValues} in order to only select
- * one value per parent among its {@code children} using the configured
- * {@code selection} type. */
- public static NumericDocValues wrap(final NumericDocValues values, Bits docsWithValue, Type selection, BitSet parents, BitSet children) {
+ /** Wraps the provided {@link NumericDocValues}, iterating over only
+ * child documents, in order to only select one value per parent among
+ * its {@code children} using the configured {@code selection} type. */
+ public static NumericDocValues wrap(final NumericDocValues values, Type selection, BitSet parents, BitSet children) {
return new NumericDocValues() {
+ private int parentDocID = -1;
+ private long value;
+
@Override
- public long get(int docID) {
- assert parents.get(docID) : "this selector may only be used on parent documents";
+ public int nextDoc() throws IOException {
- if (docID == 0) {
- // no children
- return 0;
+ if (parentDocID == -1) {
+ values.nextDoc();
}
- final int firstChild = parents.prevSetBit(docID - 1) + 1;
+ while (true) {
- long value = 0;
- boolean hasValue = false;
+ // TODO: make this crazy loop more efficient
- for (int child = children.nextSetBit(firstChild); child < docID; child = children.nextSetBit(child + 1)) {
- final long childValue = values.get(child);
- final boolean childHasValue = value != 0 || docsWithValue.get(child);
+ int childDocID = values.docID();
+ if (childDocID == NO_MORE_DOCS) {
+ parentDocID = NO_MORE_DOCS;
+ return parentDocID;
+ }
+ if (children.get(childDocID) == false) {
+ values.nextDoc();
+ continue;
+ }
+
+ assert parents.get(childDocID) == false;
+
+ parentDocID = parents.nextSetBit(childDocID);
+ value = values.longValue();
+
+ while (true) {
+ childDocID = values.nextDoc();
+ assert childDocID != parentDocID;
+ if (childDocID > parentDocID) {
+ break;
+ }
- if (hasValue == false) {
- value = childValue;
- hasValue = childHasValue;
- } else if (childHasValue) {
switch (selection) {
- case MIN:
- value = Math.min(value, childValue);
- break;
- case MAX:
- value = Math.max(value, childValue);
- break;
- default:
- throw new AssertionError();
+ case MIN:
+ value = Math.min(value, values.longValue());
+ break;
+ case MAX:
+ value = Math.max(value, values.longValue());
+ break;
+ default:
+ throw new AssertionError();
}
}
+
+ break;
+ }
+
+ return parentDocID;
+ }
+
+ @Override
+ public int advance(int targetParentDocID) throws IOException {
+ if (targetParentDocID <= parentDocID) {
+ throw new IllegalArgumentException("target must be after the current document: current=" + parentDocID + " target=" + targetParentDocID);
+ }
+
+ if (targetParentDocID == 0) {
+ return nextDoc();
+ }
+
+ int firstChild = parents.prevSetBit(targetParentDocID - 1) + 1;
+ if (values.advance(firstChild) == NO_MORE_DOCS) {
+ parentDocID = NO_MORE_DOCS;
+ return parentDocID;
+ } else {
+ return nextDoc();
}
+ }
+ @Override
+ public long longValue() {
return value;
}
+
+ @Override
+ public int docID() {
+ return parentDocID;
+ }
+ @Override
+ public long cost() {
+ return values.cost();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
index ca1337d..d54543c 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
@@ -29,7 +29,7 @@ abstract class DocValuesTermsCollector<DV> extends SimpleCollector {
@FunctionalInterface
static interface Function<R> {
- R apply(LeafReader t) throws IOException ;
+ R apply(LeafReader t) throws IOException;
}
protected DV docValues;
@@ -45,8 +45,9 @@ abstract class DocValuesTermsCollector<DV> extends SimpleCollector {
}
static Function<BinaryDocValues> binaryDocValues(String field) {
- return (ctx) -> DocValues.getBinary(ctx, field);
+ return (ctx) -> DocValues.getBinary(ctx, field);
}
+
static Function<SortedSetDocValues> sortedSetDocValues(String field) {
return (ctx) -> DocValues.getSortedSet(ctx, field);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
index ef8b3c1..3a630a1 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
@@ -43,9 +43,9 @@ interface GenericTermsCollector extends Collector {
case None:
return wrap(new TermsCollector.MV(mvFunction));
case Avg:
- return new MV.Avg(mvFunction);
+ return new MV.Avg(mvFunction);
default:
- return new MV(mvFunction, mode);
+ return new MV(mvFunction, mode);
}
}
@@ -55,13 +55,31 @@ interface GenericTermsCollector extends Collector {
return new SortedSetDocValues() {
@Override
- public void setDocument(int docID) {
- target.setDocument(docID);
- out.println("\ndoc# "+docID);
+ public int docID() {
+ return target.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ int docID = target.nextDoc();
+ out.println("\nnextDoc doc# "+docID);
+ return docID;
+ }
+
+ @Override
+ public int advance(int dest) throws IOException {
+ int docID = target.advance(dest);
+ out.println("\nadvance(" + dest + ") -> doc# "+docID);
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return target.cost();
}
@Override
- public long nextOrd() {
+ public long nextOrd() throws IOException {
return target.nextOrd();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
index d80851a..9c9072d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -26,8 +28,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.LongValues;
-import java.io.IOException;
-
/**
* A collector that collects all ordinals from a specified field matching the query.
*
@@ -77,9 +77,12 @@ final class GlobalOrdinalsCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- final long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ long segmentOrd = docTermOrds.ordValue();
+ long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
collectedOrds.set(globalOrd);
}
}
@@ -99,9 +102,11 @@ final class GlobalOrdinalsCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- collectedOrds.set(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ collectedOrds.set(docTermOrds.ordValue());
}
}
[04/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
index 2053199..a75af54 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.Set;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FilterIterator;
/**
@@ -114,13 +113,6 @@ public final class FieldFilterLeafReader extends FilterLeafReader {
return (f == null) ? null : new FieldFilterFields(f);
}
-
-
- @Override
- public NumericDocValues getNumericDocValues(String field) throws IOException {
- return hasField(field) ? super.getNumericDocValues(field) : null;
- }
-
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
return hasField(field) ? super.getBinaryDocValues(field) : null;
@@ -147,11 +139,6 @@ public final class FieldFilterLeafReader extends FilterLeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- return hasField(field) ? super.getDocsWithField(field) : null;
- }
-
- @Override
public String toString() {
final StringBuilder sb = new StringBuilder("FieldFilterLeafReader(reader=");
sb.append(in).append(", fields=");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
index ea70c98..4fd8fab 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Random;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
@@ -32,6 +31,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -40,12 +40,12 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
+import junit.framework.Assert;
+
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
-import junit.framework.Assert;
-
/**
* Utility class for sanity-checking queries.
*/
@@ -246,11 +246,6 @@ public class QueryUtils {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- return null;
- }
-
- @Override
public NumericDocValues getNormValues(String field) throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 98cd2a7..e62c563 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -144,6 +144,7 @@ import junit.framework.AssertionFailedError;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/**
* Base class for all Lucene unit tests, Junit3 or Junit4 variant.
@@ -2440,8 +2441,8 @@ public abstract class LuceneTestCase extends Assert {
if (leftValues != null && rightValues != null) {
assertDocValuesEquals(info, leftReader.maxDoc(), leftValues, rightValues);
} else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
+ assertTrue(info + ": left numeric doc values for field=\"" + field + "\" are not null", leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS);
+ assertTrue(info + ": right numeric doc values for field=\"" + field + "\" are not null", rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS);
}
}
@@ -2449,14 +2450,17 @@ public abstract class LuceneTestCase extends Assert {
BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field);
BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field);
if (leftValues != null && rightValues != null) {
- for(int docID=0;docID<leftReader.maxDoc();docID++) {
- final BytesRef left = BytesRef.deepCopyOf(leftValues.get(docID));
- final BytesRef right = rightValues.get(docID);
- assertEquals(info, left, right);
+ while (true) {
+ int docID = leftValues.nextDoc();
+ assertEquals(docID, rightValues.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ assertEquals(leftValues.binaryValue(), rightValues.binaryValue());
}
} else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
+ assertTrue(info, leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS);
+ assertTrue(info, rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS);
}
}
@@ -2474,8 +2478,10 @@ public abstract class LuceneTestCase extends Assert {
}
// bytes
for(int docID=0;docID<leftReader.maxDoc();docID++) {
- final BytesRef left = BytesRef.deepCopyOf(leftValues.get(docID));
- final BytesRef right = rightValues.get(docID);
+ assertEquals(docID, leftValues.nextDoc());
+ assertEquals(docID, rightValues.nextDoc());
+ final BytesRef left = BytesRef.deepCopyOf(leftValues.binaryValue());
+ final BytesRef right = rightValues.binaryValue();
assertEquals(info, left, right);
}
} else {
@@ -2497,9 +2503,12 @@ public abstract class LuceneTestCase extends Assert {
assertEquals(info, left, right);
}
// ord lists
- for(int docID=0;docID<leftReader.maxDoc();docID++) {
- leftValues.setDocument(docID);
- rightValues.setDocument(docID);
+ while (true) {
+ int docID = leftValues.nextDoc();
+ assertEquals(docID, rightValues.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
long ord;
while ((ord = leftValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
assertEquals(info, ord, rightValues.nextOrd());
@@ -2516,46 +2525,36 @@ public abstract class LuceneTestCase extends Assert {
SortedNumericDocValues leftValues = MultiDocValues.getSortedNumericValues(leftReader, field);
SortedNumericDocValues rightValues = MultiDocValues.getSortedNumericValues(rightReader, field);
if (leftValues != null && rightValues != null) {
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- leftValues.setDocument(i);
- long expected[] = new long[leftValues.count()];
- for (int j = 0; j < expected.length; j++) {
- expected[j] = leftValues.valueAt(j);
+ while (true) {
+ int docID = leftValues.nextDoc();
+ assertEquals(docID, rightValues.nextDoc());
+ if (docID == NO_MORE_DOCS) {
+ break;
}
- rightValues.setDocument(i);
- for (int j = 0; j < expected.length; j++) {
- assertEquals(info, expected[j], rightValues.valueAt(j));
+ assertEquals(info, leftValues.docValueCount(), rightValues.docValueCount());
+ for (int j = 0; j < leftValues.docValueCount(); j++) {
+ assertEquals(info, leftValues.nextValue(), rightValues.nextValue());
}
- assertEquals(info, expected.length, rightValues.count());
}
} else {
assertNull(info, leftValues);
assertNull(info, rightValues);
}
}
-
- {
- Bits leftBits = MultiDocValues.getDocsWithField(leftReader, field);
- Bits rightBits = MultiDocValues.getDocsWithField(rightReader, field);
- if (leftBits != null && rightBits != null) {
- assertEquals(info, leftBits.length(), rightBits.length());
- for (int i = 0; i < leftBits.length(); i++) {
- assertEquals(info, leftBits.get(i), rightBits.get(i));
- }
- } else {
- assertNull(info, leftBits);
- assertNull(info, rightBits);
- }
- }
}
}
public void assertDocValuesEquals(String info, int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) throws IOException {
assertNotNull(info, leftDocValues);
assertNotNull(info, rightDocValues);
- for(int docID=0;docID<num;docID++) {
- assertEquals(leftDocValues.get(docID),
- rightDocValues.get(docID));
+ while (true) {
+ int leftDocID = leftDocValues.nextDoc();
+ int rightDocID = rightDocValues.nextDoc();
+ assertEquals(leftDocID, rightDocID);
+ if (leftDocID == NO_MORE_DOCS) {
+ return;
+ }
+ assertEquals(leftDocValues.longValue(), rightDocValues.longValue());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
index 19fcb3b..710e505 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
@@ -377,8 +377,7 @@ public final class TestUtil {
if (reader.getBinaryDocValues(info.name) != null ||
reader.getNumericDocValues(info.name) != null ||
reader.getSortedDocValues(info.name) != null ||
- reader.getSortedSetDocValues(info.name) != null ||
- reader.getDocsWithField(info.name) != null) {
+ reader.getSortedSetDocValues(info.name) != null) {
throw new RuntimeException("field: " + info.name + " has docvalues but should omit them!");
}
break;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java
index 2bcaa33..ab36307 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java
@@ -22,13 +22,12 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.analytics.accumulator.FacetingAccumulator;
import org.apache.solr.analytics.accumulator.ValueAccumulator;
-import org.apache.solr.analytics.util.AnalyticsParsers;
import org.apache.solr.analytics.util.AnalyticsParsers.NumericParser;
import org.apache.solr.analytics.util.AnalyticsParsers.Parser;
+import org.apache.solr.analytics.util.AnalyticsParsers;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.DateValueFieldType;
import org.apache.solr.schema.SchemaField;
@@ -48,9 +47,8 @@ public class FieldFacetAccumulator extends ValueAccumulator {
protected final boolean numField;
protected final boolean dateField;
protected SortedSetDocValues setValues;
- protected SortedDocValues sortValues;
- protected NumericDocValues numValues;
- protected Bits numValuesBits;
+ protected SortedDocValues sortValues;
+ protected NumericDocValues numValues;
public FieldFacetAccumulator(SolrIndexSearcher searcher, FacetValueAccumulator parent, SchemaField schemaField) throws IOException {
if( !schemaField.hasDocValues() ){
@@ -80,7 +78,6 @@ public class FieldFacetAccumulator extends ValueAccumulator {
} else {
if (numField) {
numValues = context.reader().getNumericDocValues(name);
- numValuesBits = context.reader().getDocsWithField(name);
} else {
sortValues = context.reader().getSortedDocValues(name);
}
@@ -96,12 +93,16 @@ public class FieldFacetAccumulator extends ValueAccumulator {
if (multiValued) {
boolean exists = false;
if (setValues!=null) {
- setValues.setDocument(doc);
- int term;
- while ((term = (int)setValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- exists = true;
- final BytesRef value = setValues.lookupOrd(term);
- parent.collectField(doc, name, parser.parse(value) );
+ if (doc > setValues.docID()) {
+ setValues.advance(doc);
+ }
+ if (doc == setValues.docID()) {
+ int term;
+ while ((term = (int)setValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ exists = true;
+ final BytesRef value = setValues.lookupOrd(term);
+ parent.collectField(doc, name, parser.parse(value) );
+ }
}
}
if (!exists) {
@@ -110,9 +111,12 @@ public class FieldFacetAccumulator extends ValueAccumulator {
} else {
if(numField){
if(numValues != null) {
- long v = numValues.get(doc);
- if( v != 0 || numValuesBits.get(doc) ){
- parent.collectField(doc, name, ((NumericParser)parser).parseNum(v));
+ int valuesDocID = numValues.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = numValues.advance(doc);
+ }
+ if (valuesDocID == doc) {
+ parent.collectField(doc, name, ((NumericParser)parser).parseNum(numValues.longValue()));
} else {
parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );
}
@@ -121,11 +125,13 @@ public class FieldFacetAccumulator extends ValueAccumulator {
}
} else {
if(sortValues != null) {
- final int ord = sortValues.getOrd(doc);
- if (ord < 0) {
- parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );
+ if (doc > sortValues.docID()) {
+ sortValues.advance(doc);
+ }
+ if (doc == sortValues.docID()) {
+ parent.collectField(doc, name, parser.parse(sortValues.lookupOrd(sortValues.ordValue())) );
} else {
- parent.collectField(doc, name, parser.parse(sortValues.lookupOrd(ord)) );
+ parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );
}
} else {
parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/AbstractDelegatingStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/AbstractDelegatingStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/AbstractDelegatingStatsCollector.java
index 45dfa00..92969f1 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/AbstractDelegatingStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/AbstractDelegatingStatsCollector.java
@@ -64,7 +64,7 @@ public abstract class AbstractDelegatingStatsCollector implements StatsCollector
return delegate.getFunction();
}
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
delegate.collect(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MedianStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MedianStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MedianStatsCollector.java
index c91840b..bf71429 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MedianStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MedianStatsCollector.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.analytics.statistics;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -52,7 +53,7 @@ public class MedianStatsCollector extends AbstractDelegatingStatsCollector{
}
@Override
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
super.collect(doc);
if (value.exists) {
values.add(function.doubleVal(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
index 7e3bde1..c21b045 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
@@ -51,7 +51,7 @@ public class MinMaxStatsCollector implements StatsCollector{
value = valueFiller.getValue();
}
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
valueFiller.fillValue(doc);
if( value.exists ){
valueCount += 1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
index 6a64eeb..1f22baa 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.analytics.statistics;
+import java.io.IOException;
import java.util.Set;
import org.apache.lucene.queries.function.ValueSource;
@@ -33,7 +34,7 @@ public class NumericStatsCollector extends MinMaxStatsCollector {
super(source, statsList);
}
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
super.collect(doc);
double value = function.doubleVal(doc);
sum += value;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/PercentileStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/PercentileStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/PercentileStatsCollector.java
index 5f63ef9..e12cb83 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/PercentileStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/PercentileStatsCollector.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.analytics.statistics;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
@@ -69,7 +70,7 @@ public class PercentileStatsCollector extends AbstractDelegatingStatsCollector{
return PercentileCalculator.getPercentiles(values, percentiles);
}
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
super.collect(doc);
if (value.exists) {
values.add((Comparable)value.toObject());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollector.java
index d9a4640..039300f 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollector.java
@@ -33,7 +33,7 @@ public interface StatsCollector {
* Collect values from the value source and add to statistics.
* @param doc Document to collect from
*/
- void collect(int doc);
+ void collect(int doc) throws IOException;
/**
* @param context The context to read documents from.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/UniqueStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/UniqueStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/UniqueStatsCollector.java
index d4d537c..461b0f4 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/UniqueStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/UniqueStatsCollector.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.analytics.statistics;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
@@ -30,7 +31,7 @@ public class UniqueStatsCollector extends AbstractDelegatingStatsCollector{
}
@Override
- public void collect(int doc) {
+ public void collect(int doc) throws IOException {
super.collect(doc);
if (value.exists) {
uniqueValues.add(value.toObject());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AbsoluteValueDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AbsoluteValueDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AbsoluteValueDoubleFunction.java
index 62c99a7..f1e996c 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AbsoluteValueDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AbsoluteValueDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -39,7 +41,7 @@ public class AbsoluteValueDoubleFunction extends SingleDoubleFunction {
return name()+"("+source.description()+")";
}
- protected double func(int doc, FunctionValues vals) {
+ protected double func(int doc, FunctionValues vals) throws IOException {
double d = vals.doubleVal(doc);
if (d<0) {
return d*-1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AddDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AddDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AddDoubleFunction.java
index 1a8a254..c47a0a3 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AddDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/AddDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -36,7 +38,7 @@ public class AddDoubleFunction extends MultiDoubleFunction {
}
@Override
- protected double func(int doc, FunctionValues[] valsArr) {
+ protected double func(int doc, FunctionValues[] valsArr) throws IOException {
double sum = 0d;
for (FunctionValues val : valsArr) {
sum += val.doubleVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConcatStringFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConcatStringFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConcatStringFunction.java
index 3620c47..b1e1dc0 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConcatStringFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConcatStringFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -36,7 +38,7 @@ public class ConcatStringFunction extends MultiStringFunction {
}
@Override
- protected String func(int doc, FunctionValues[] valsArr) {
+ protected String func(int doc, FunctionValues[] valsArr) throws IOException {
StringBuilder sb = new StringBuilder();
for (FunctionValues val : valsArr) {
String v = val.strVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java
index 22dde4c..d13795d 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java
@@ -28,7 +28,6 @@ import org.apache.lucene.legacy.LegacyNumericUtils;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.LongDocValues;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueDate;
@@ -59,26 +58,39 @@ public class DateFieldSource extends LongFieldSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new LongDocValues(this) {
+
+ private long getDocValue(int doc) throws IOException {
+ int arrDocID = arr.docID();
+ if (arrDocID < doc) {
+ arrDocID = arr.advance(doc);
+ }
+ if (arrDocID == doc) {
+ return arr.longValue();
+ } else {
+ return 0;
+ }
+ }
+
@Override
- public long longVal(int doc) {
- return arr.get(doc);
+ public long longVal(int doc) throws IOException {
+ return getDocValue(doc);
}
@Override
- public boolean exists(int doc) {
- return valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getDocValue(doc);
+ return arr.docID() == doc;
}
@Override
- public Object objectVal(int doc) {
- return exists(doc) ? longToObject(arr.get(doc)) : null;
+ public Object objectVal(int doc) throws IOException {
+ return exists(doc) ? longToObject(getDocValue(doc)) : null;
}
@Override
- public String strVal(int doc) {
- return exists(doc) ? longToString(arr.get(doc)) : null;
+ public String strVal(int doc) throws IOException {
+ return exists(doc) ? longToString(getDocValue(doc)) : null;
}
@Override
@@ -92,8 +104,8 @@ public class DateFieldSource extends LongFieldSource {
}
@Override
- public void fillValue(int doc) {
- mval.value = arr.get(doc);
+ public void fillValue(int doc) throws IOException {
+ mval.value = getDocValue(doc);
mval.exists = exists(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateMathFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateMathFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateMathFunction.java
index 2bf40f3..77b172a 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateMathFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateMathFunction.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
import java.text.ParseException;
import java.util.Date;
@@ -50,7 +51,7 @@ public class DateMathFunction extends MultiDateFunction {
}
@Override
- protected long func(int doc, FunctionValues[] valsArr) {
+ protected long func(int doc, FunctionValues[] valsArr) throws IOException {
long time = 0;
Date date = (Date)valsArr[0].objectVal(doc);
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DivDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DivDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DivDoubleFunction.java
index 5e43901..25c4c96 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DivDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DivDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -39,7 +41,7 @@ public class DivDoubleFunction extends DualDoubleFunction {
}
@Override
- protected double func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected double func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.doubleVal(doc)/bVals.doubleVal(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DualDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DualDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DualDoubleFunction.java
index 96058aa..97b8af7 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DualDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DualDoubleFunction.java
@@ -39,7 +39,7 @@ public abstract class DualDoubleFunction extends ValueSource {
}
protected abstract String name();
- protected abstract double func(int doc, FunctionValues aVals, FunctionValues bVals);
+ protected abstract double func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException;
@Override
public String description() {
@@ -52,17 +52,17 @@ public abstract class DualDoubleFunction extends ValueSource {
final FunctionValues bVals = b.getValues(context, readerContext);
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return func(doc, aVals, bVals);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return aVals.exists(doc) & bVals.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java
index 22f5a57..3f374dc 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java
@@ -60,53 +60,53 @@ public class FilterFieldSource extends ValueSource {
return new FunctionValues() {
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return vals.byteVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return vals.shortVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return vals.floatVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return vals.intVal(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return vals.longVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return vals.doubleVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return vals.strVal(doc);
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc)? vals.objectVal(doc) : null;
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
Object other = vals.objectVal(doc);
return other!=null&&!missValue.equals(other);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return NAME + '(' + vals.toString(doc) + ')';
}
@@ -122,7 +122,7 @@ public class FilterFieldSource extends ValueSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
delegateFiller.fillValue(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/LogDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/LogDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/LogDoubleFunction.java
index 63906de..41a6b67 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/LogDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/LogDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -35,7 +37,7 @@ public class LogDoubleFunction extends DualDoubleFunction {
}
@Override
- protected double func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected double func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return Math.log(aVals.doubleVal(doc))/Math.log(bVals.doubleVal(doc));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDateFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDateFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDateFunction.java
index 3daa0ea..beeaaa7 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDateFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDateFunction.java
@@ -39,7 +39,7 @@ public abstract class MultiDateFunction extends ValueSource {
}
abstract protected String name();
- abstract protected long func(int doc, FunctionValues[] valsArr);
+ abstract protected long func(int doc, FunctionValues[] valsArr) throws IOException;
@Override
public String description() {
@@ -67,12 +67,12 @@ public abstract class MultiDateFunction extends ValueSource {
return new LongDocValues(this) {
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return func(doc, valsArr);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
boolean exists = true;
for (FunctionValues val : valsArr) {
exists = exists & val.exists(doc);
@@ -81,7 +81,7 @@ public abstract class MultiDateFunction extends ValueSource {
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
boolean firstTime=true;
@@ -108,7 +108,7 @@ public abstract class MultiDateFunction extends ValueSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = longVal(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDoubleFunction.java
index ded716c..09956f2 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiDoubleFunction.java
@@ -38,7 +38,7 @@ public abstract class MultiDoubleFunction extends ValueSource {
}
abstract protected String name();
- abstract protected double func(int doc, FunctionValues[] valsArr);
+ abstract protected double func(int doc, FunctionValues[] valsArr) throws IOException;
@Override
public String description() {
@@ -66,12 +66,12 @@ public abstract class MultiDoubleFunction extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return func(doc, valsArr);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
boolean exists = true;
for (FunctionValues val : valsArr) {
exists = exists & val.exists(doc);
@@ -80,7 +80,7 @@ public abstract class MultiDoubleFunction extends ValueSource {
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
boolean firstTime=true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java
index 1d9c67d..4689d8b 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java
@@ -40,7 +40,7 @@ public abstract class MultiStringFunction extends ValueSource {
}
abstract protected String name();
- abstract protected CharSequence func(int doc, FunctionValues[] valsArr);
+ abstract protected CharSequence func(int doc, FunctionValues[] valsArr) throws IOException;
@Override
public String description() {
@@ -68,13 +68,13 @@ public abstract class MultiStringFunction extends ValueSource {
return new StrDocValues(this) {
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
CharSequence cs = func(doc, valsArr);
return cs != null ? cs.toString() : null;
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
boolean exists = true;
for (FunctionValues val : valsArr) {
exists = exists & val.exists(doc);
@@ -83,7 +83,7 @@ public abstract class MultiStringFunction extends ValueSource {
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder bytes) {
+ public boolean bytesVal(int doc, BytesRefBuilder bytes) throws IOException {
bytes.clear();
CharSequence cs = func(doc, valsArr);
if( cs != null ){
@@ -95,7 +95,7 @@ public abstract class MultiStringFunction extends ValueSource {
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
boolean firstTime=true;
@@ -122,7 +122,7 @@ public abstract class MultiStringFunction extends ValueSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.exists = bytesVal(doc, mval.value);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiplyDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiplyDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiplyDoubleFunction.java
index 1624ae9..83f26f0 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiplyDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/MultiplyDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -36,7 +38,7 @@ public class MultiplyDoubleFunction extends MultiDoubleFunction {
}
@Override
- protected double func(int doc, FunctionValues[] valsArr) {
+ protected double func(int doc, FunctionValues[] valsArr) throws IOException {
double product = 1d;
for (FunctionValues val : valsArr) {
product *= val.doubleVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/NegateDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/NegateDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/NegateDoubleFunction.java
index 4297261..9c8445d 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/NegateDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/NegateDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -39,7 +41,7 @@ public class NegateDoubleFunction extends SingleDoubleFunction {
return name()+"("+source.description()+")";
}
- protected double func(int doc, FunctionValues vals) {
+ protected double func(int doc, FunctionValues vals) throws IOException {
return vals.doubleVal(doc)*-1;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/PowDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/PowDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/PowDoubleFunction.java
index 757e31b..043313b 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/PowDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/PowDoubleFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -40,7 +42,7 @@ public class PowDoubleFunction extends DualDoubleFunction {
}
@Override
- protected double func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected double func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return Math.pow(aVals.doubleVal(doc), bVals.doubleVal(doc));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ReverseStringFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ReverseStringFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ReverseStringFunction.java
index 68a18d4..4fa0f99 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ReverseStringFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ReverseStringFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.analytics.util.valuesource;
+import java.io.IOException;
+
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,7 +37,7 @@ public class ReverseStringFunction extends SingleStringFunction {
return NAME;
}
- protected CharSequence func(int doc, FunctionValues vals) {
+ protected CharSequence func(int doc, FunctionValues vals) throws IOException {
String val = vals.strVal(doc);
return val != null ? StringUtils.reverse(val) : null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleDoubleFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleDoubleFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleDoubleFunction.java
index 967f014..04f79ed 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleDoubleFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleDoubleFunction.java
@@ -41,24 +41,24 @@ public abstract class SingleDoubleFunction extends ValueSource {
}
abstract String name();
- abstract double func(int doc, FunctionValues vals);
+ abstract double func(int doc, FunctionValues vals) throws IOException;
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FunctionValues vals = source.getValues(context, readerContext);
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return func(doc, vals);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return vals.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + vals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java
index b6a79f0..b3357f9 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java
@@ -44,20 +44,20 @@ public abstract class SingleStringFunction extends ValueSource {
}
abstract String name();
- abstract CharSequence func(int doc, FunctionValues vals);
+ abstract CharSequence func(int doc, FunctionValues vals) throws IOException;
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FunctionValues vals = source.getValues(context, readerContext);
return new StrDocValues(this) {
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
CharSequence cs = func(doc, vals);
return cs != null ? cs.toString() : null;
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder bytes) {
+ public boolean bytesVal(int doc, BytesRefBuilder bytes) throws IOException {
CharSequence cs = func(doc, vals);
if( cs != null ){
bytes.copyChars(func(doc,vals));
@@ -69,17 +69,17 @@ public abstract class SingleStringFunction extends ValueSource {
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return strVal(doc);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return vals.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + strVal(doc) + ')';
}
@@ -94,7 +94,7 @@ public abstract class SingleStringFunction extends ValueSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.exists = bytesVal(doc, mval.value);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index eb22d90..e307e70 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -275,7 +275,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
SortedDocValues[] sortedDocValues = null;
LongValues segmentOrdinalMap = null;
SortedDocValues currentValues = null;
- if(values instanceof MultiDocValues.MultiSortedDocValues) {
+ if(values instanceof MultiDocValues.MultiSortedDocValues) {
ordinalMap = ((MultiDocValues.MultiSortedDocValues)values).mapping;
sortedDocValues = ((MultiDocValues.MultiSortedDocValues)values).values;
currentValues = sortedDocValues[currentContext];
@@ -299,8 +299,11 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
int contextDoc = globalDoc - currentDocBase;
if(ordinalMap != null) {
- int ord = currentValues.getOrd(contextDoc);
- if(ord > -1) {
+ if (contextDoc > currentValues.docID()) {
+ currentValues.advance(contextDoc);
+ }
+ if (contextDoc == currentValues.docID()) {
+ int ord = currentValues.ordValue();
++count;
BytesRef ref = currentValues.lookupOrd(ord);
ord = (int)segmentOrdinalMap.get(ord);
@@ -309,8 +312,11 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
collapsedSet.add(globalDoc);
}
} else {
- int ord = values.getOrd(globalDoc);
- if(ord > -1) {
+ if (globalDoc > values.docID()) {
+ values.advance(globalDoc);
+ }
+ if (globalDoc == values.docID()) {
+ int ord = values.ordValue();
++count;
BytesRef ref = values.lookupOrd(ord);
ordBytes.put(ord, BytesRef.deepCopyOf(ref));
@@ -340,7 +346,16 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
}
int contextDoc = globalDoc - currentDocBase;
- long value = collapseValues.get(contextDoc);
+ int valueDocID = collapseValues.docID();
+ if (valueDocID < contextDoc) {
+ valueDocID = collapseValues.advance(contextDoc);
+ }
+ long value;
+ if (valueDocID == contextDoc) {
+ value = collapseValues.longValue();
+ } else {
+ value = 0;
+ }
if(value != nullValue) {
++count;
groupSet.add(value);
@@ -361,6 +376,16 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
Collector groupExpandCollector = null;
if(values != null) {
+ //Get The Top Level SortedDocValues again so we can re-iterate:
+ if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
+ Map<String, UninvertingReader.Type> mapping = new HashMap();
+ mapping.put(field, UninvertingReader.Type.SORTED);
+ UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getLeafReader(), field), mapping);
+ values = uninvertingReader.getSortedDocValues(field);
+ } else {
+ values = DocValues.getSorted(reader, field);
+ }
+
groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
} else {
groupExpandCollector = new NumericGroupExpandCollector(field, nullValue, groupSet, collapsedSet, limit, sort);
@@ -452,7 +477,6 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
if (!rb.doExpand) {
return;
}
-
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
SolrQueryRequest req = rb.req;
NamedList expanded = (NamedList) req.getContext().get("expanded");
@@ -553,12 +577,23 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
int globalDoc = docId + docBase;
int ord = -1;
if(ordinalMap != null) {
- ord = segmentValues.getOrd(docId);
- if(ord > -1) {
- ord = (int)segmentOrdinalMap.get(ord);
+ if (docId > segmentValues.docID()) {
+ segmentValues.advance(docId);
+ }
+ if (docId == segmentValues.docID()) {
+ ord = (int)segmentOrdinalMap.get(segmentValues.ordValue());
+ } else {
+ ord = -1;
}
} else {
- ord = docValues.getOrd(globalDoc);
+ if (globalDoc > docValues.docID()) {
+ docValues.advance(globalDoc);
+ }
+ if (globalDoc == docValues.docID()) {
+ ord = docValues.ordValue();
+ } else {
+ ord = -1;
+ }
}
if (ord > -1 && groupBits.get(ord) && !collapsedSet.contains(globalDoc)) {
@@ -624,7 +659,16 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
@Override
public void collect(int docId) throws IOException {
- long value = docValues.get(docId);
+ int valuesDocID = docValues.docID();
+ if (valuesDocID < docId) {
+ valuesDocID = docValues.advance(docId);
+ }
+ long value;
+ if (valuesDocID == docId) {
+ value = docValues.longValue();
+ } else {
+ value = 0;
+ }
final int index;
if (value != nullValue &&
(index = leafCollectors.indexOf(value)) >= 0 &&
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
index 6df92c0..5a8f544 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
@@ -22,9 +22,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -97,9 +97,17 @@ public class FieldFacetStats {
if (topLevelSortedValues == null) {
topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
}
+
+ if (docID > topLevelSortedValues.docID()) {
+ topLevelSortedValues.advance(docID);
+ }
-
- int term = topLevelSortedValues.getOrd(docID);
+ int term;
+ if (docID == topLevelSortedValues.docID()) {
+ term = topLevelSortedValues.ordValue();
+ } else {
+ term = -1;
+ }
int arrIdx = term;
if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
@@ -162,8 +170,12 @@ public class FieldFacetStats {
topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
}
- int ord = topLevelSortedValues.getOrd(docID);
- if (ord != -1) {
+ if (docID > topLevelSortedValues.docID()) {
+ topLevelSortedValues.advance(docID);
+ }
+
+ if (docID == topLevelSortedValues.docID()) {
+ int ord = topLevelSortedValues.ordValue();
Integer missingCount = missingStats.get(ord);
if (missingCount == null) {
missingStats.put(ord, 1);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
index 8c85fda..ae24568 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
@@ -39,7 +39,7 @@ public interface StatsValues {
/** Accumulate the value associated with <code>docID</code>.
* @see #setNextReader(org.apache.lucene.index.LeafReaderContext) */
- void accumulate(int docID);
+ void accumulate(int docID) throws IOException;
/**
* Accumulate the values based on the given value
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
index a2e4a4e..8a35ee0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
@@ -485,7 +485,7 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
}
@Override
- public void accumulate(int docID) {
+ public void accumulate(int docID) throws IOException {
if (values.exists(docID)) {
Number value = (Number) values.objectVal(docID);
accumulate(value, 1);
@@ -640,7 +640,7 @@ class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
* {@inheritDoc}
*/
@Override
- public void accumulate(int docID) {
+ public void accumulate(int docID) throws IOException {
if (values.exists(docID)) {
Integer intValue = (Integer) values.objectVal(docID);
String stringValue = values.strVal(docID);
@@ -720,7 +720,7 @@ class DateStatsValues extends AbstractStatsValues<Date> {
}
@Override
- public void accumulate(int docID) {
+ public void accumulate(int docID) throws IOException {
if (values.exists(docID)) {
accumulate((Date) values.objectVal(docID), 1);
} else {
@@ -824,7 +824,7 @@ class StringStatsValues extends AbstractStatsValues<String> {
}
@Override
- public void accumulate(int docID) {
+ public void accumulate(int docID) throws IOException {
if (values.exists(docID)) {
String value = values.strVal(docID);
if (value != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
index 0502739..7c7098c 100644
--- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
+++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
@@ -20,29 +20,10 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CompositeReader;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
-import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Bits;
@@ -120,12 +101,6 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- ensureOpen();
- return MultiDocValues.getDocsWithField(in, field);
- }
-
- @Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
return MultiDocValues.getBinaryValues(in, field);
@@ -158,6 +133,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
int size = in.leaves().size();
final SortedDocValues[] values = new SortedDocValues[size];
final int[] starts = new int[size+1];
+ long totalCost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = in.leaves().get(i);
final LeafReader reader = context.reader();
@@ -169,11 +145,12 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
if (v == null) {
v = DocValues.emptySorted();
}
+ totalCost += v.cost();
values[i] = v;
starts[i] = context.docBase;
}
starts[size] = maxDoc();
- return new MultiSortedDocValues(values, starts, map);
+ return new MultiSortedDocValues(values, starts, map, totalCost);
}
@Override
@@ -185,8 +162,8 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
if (map == null) {
// uncached, or not a multi dv
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(in, field);
- if (dv instanceof MultiSortedSetDocValues) {
- map = ((MultiSortedSetDocValues)dv).mapping;
+ if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
+ map = ((MultiDocValues.MultiSortedSetDocValues)dv).mapping;
if (map.owner == getCoreCacheKey() && merging == false) {
cachedOrdMaps.put(field, map);
}
@@ -199,6 +176,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
int size = in.leaves().size();
final SortedSetDocValues[] values = new SortedSetDocValues[size];
final int[] starts = new int[size+1];
+ long cost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = in.leaves().get(i);
final LeafReader reader = context.reader();
@@ -212,9 +190,10 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
values[i] = v;
starts[i] = context.docBase;
+ cost += v.cost();
}
starts[size] = maxDoc();
- return new MultiSortedSetDocValues(values, starts, map);
+ return new MultiDocValues.MultiSortedSetDocValues(values, starts, map, cost);
}
// TODO: this could really be a weak map somewhere else on the coreCacheKey,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java b/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
index 9b281a9..b21764d 100644
--- a/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
@@ -21,7 +21,7 @@ import java.util.List;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
@@ -70,14 +70,14 @@ public class DocValuesFacets {
OrdinalMap ordinalMap = null; // for mapping per-segment ords to global ones
if (multiValued) {
si = searcher.getLeafReader().getSortedSetDocValues(fieldName);
- if (si instanceof MultiSortedSetDocValues) {
+ if (si instanceof MultiDocValues.MultiSortedSetDocValues) {
ordinalMap = ((MultiSortedSetDocValues)si).mapping;
}
} else {
SortedDocValues single = searcher.getLeafReader().getSortedDocValues(fieldName);
si = single == null ? null : DocValues.singleton(single);
- if (single instanceof MultiSortedDocValues) {
- ordinalMap = ((MultiSortedDocValues)single).mapping;
+ if (single instanceof MultiDocValues.MultiSortedDocValues) {
+ ordinalMap = ((MultiDocValues.MultiSortedDocValues)single).mapping;
}
}
if (si == null) {
@@ -272,7 +272,15 @@ public class DocValuesFacets {
final LongValues ordmap = map == null ? null : map.getGlobalOrds(subIndex);
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- int term = si.getOrd(doc);
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ int term;
+ if (doc == si.docID()) {
+ term = si.ordValue();
+ } else {
+ term = -1;
+ }
if (map != null && term >= 0) {
term = (int) ordmap.get(term);
}
@@ -293,7 +301,14 @@ public class DocValuesFacets {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- segCounts[1+si.getOrd(doc)]++;
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ if (doc == si.docID()) {
+ segCounts[1+si.ordValue()]++;
+ } else {
+ segCounts[0]++;
+ }
}
// migrate to global ords (if necessary)
@@ -319,23 +334,22 @@ public class DocValuesFacets {
final LongValues ordMap = map == null ? null : map.getGlobalOrds(subIndex);
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- si.setDocument(doc);
- // strange do-while to collect the missing count (first ord is NO_MORE_ORDS)
- int term = (int) si.nextOrd();
- if (term < 0) {
- if (startTermIndex == -1) {
- counts[0]++; // missing count
- }
- continue;
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ if (doc == si.docID()) {
+ // strange do-while to collect the missing count (first ord is NO_MORE_ORDS)
+ int term = (int) si.nextOrd();
+ do {
+ if (map != null) {
+ term = (int) ordMap.get(term);
+ }
+ int arrIdx = term-startTermIndex;
+ if (arrIdx>=0 && arrIdx<counts.length) counts[arrIdx]++;
+ } while ((term = (int) si.nextOrd()) >= 0);
+ } else if (startTermIndex == -1) {
+ counts[0]++; // missing count
}
-
- do {
- if (map != null) {
- term = (int) ordMap.get(term);
- }
- int arrIdx = term-startTermIndex;
- if (arrIdx>=0 && arrIdx<counts.length) counts[arrIdx]++;
- } while ((term = (int) si.nextOrd()) >= 0);
}
}
@@ -351,14 +365,16 @@ public class DocValuesFacets {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- si.setDocument(doc);
- int term = (int) si.nextOrd();
- if (term < 0) {
- counts[0]++; // missing
- } else {
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ if (doc == si.docID()) {
+ int term = (int) si.nextOrd();
do {
segCounts[1+term]++;
} while ((term = (int)si.nextOrd()) >= 0);
+ } else {
+ counts[0]++; // missing
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/request/DocValuesStats.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/DocValuesStats.java b/solr/core/src/java/org/apache/solr/request/DocValuesStats.java
index 4001f5a..efc5cf7 100644
--- a/solr/core/src/java/org/apache/solr/request/DocValuesStats.java
+++ b/solr/core/src/java/org/apache/solr/request/DocValuesStats.java
@@ -20,9 +20,9 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
@@ -86,13 +86,13 @@ public class DocValuesStats {
si = searcher.getLeafReader().getSortedSetDocValues(fieldName);
if (si instanceof MultiSortedSetDocValues) {
- ordinalMap = ((MultiSortedSetDocValues)si).mapping;
+ ordinalMap = ((MultiDocValues.MultiSortedSetDocValues)si).mapping;
}
} else {
SortedDocValues single = searcher.getLeafReader().getSortedDocValues(fieldName);
si = single == null ? null : DocValues.singleton(single);
- if (single instanceof MultiSortedDocValues) {
- ordinalMap = ((MultiSortedDocValues)single).mapping;
+ if (single instanceof MultiDocValues.MultiSortedDocValues) {
+ ordinalMap = ((MultiDocValues.MultiSortedDocValues)single).mapping;
}
}
if (si == null) {
@@ -127,7 +127,7 @@ public class DocValuesStats {
if (sub == null) {
sub = DocValues.emptySortedSet();
}
- final SortedDocValues singleton = DocValues.unwrapSingleton(sub);
+ SortedDocValues singleton = DocValues.unwrapSingleton(sub);
if (singleton != null) {
// some codecs may optimize SORTED_SET storage for single-valued fields
missingDocCountTotal += accumSingle(counts, docBase, facetStats, singleton, disi, subIndex, ordinalMap);
@@ -174,8 +174,11 @@ public class DocValuesStats {
int missingDocCount = 0;
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- int term = si.getOrd(doc);
- if (term >= 0) {
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ if (doc == si.docID()) {
+ int term = si.ordValue();
if (map != null) {
term = (int) ordMap.get(term);
}
@@ -201,21 +204,22 @@ public class DocValuesStats {
int missingDocCount = 0;
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- si.setDocument(doc);
- long ord;
- boolean emptyTerm = true;
- while ((ord = si.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- emptyTerm = false;
- int term = (int) ord;
- if (map != null) {
- term = (int) ordMap.get(term);
- }
- counts[term]++;
- for (FieldFacetStats f : facetStats) {
- f.facetTermNum(docBase + doc, term);
- }
+ if (doc > si.docID()) {
+ si.advance(doc);
}
- if (emptyTerm){
+ if (doc == si.docID()) {
+ long ord;
+ while ((ord = si.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ int term = (int) ord;
+ if (map != null) {
+ term = (int) ordMap.get(term);
+ }
+ counts[term]++;
+ for (FieldFacetStats f : facetStats) {
+ f.facetTermNum(docBase + doc, term);
+ }
+ }
+ } else {
for (FieldFacetStats f : facetStats) {
f.facetMissingNum(docBase + doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/request/IntervalFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java
index db26e12..6187feb 100644
--- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java
@@ -25,12 +25,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
-import org.apache.lucene.legacy.LegacyNumericType;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FilterNumericDocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.legacy.LegacyNumericType;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
@@ -182,7 +183,6 @@ public class IntervalFacets implements Iterable<FacetInterval> {
final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
LeafReaderContext ctx = null;
NumericDocValues longs = null;
- Bits docsWithField = null;
for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
@@ -198,24 +198,22 @@ public class IntervalFacets implements Iterable<FacetInterval> {
longs = DocValues.getNumeric(ctx.reader(), fieldName);
break;
case FLOAT:
- final NumericDocValues floats = DocValues.getNumeric(ctx.reader(), fieldName);
// TODO: this bit flipping should probably be moved to tie-break in the PQ comparator
- longs = new NumericDocValues() {
+ longs = new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) {
@Override
- public long get(int docID) {
- long bits = floats.get(docID);
+ public long longValue() {
+ long bits = super.longValue();
if (bits < 0) bits ^= 0x7fffffffffffffffL;
return bits;
}
};
break;
case DOUBLE:
- final NumericDocValues doubles = DocValues.getNumeric(ctx.reader(), fieldName);
// TODO: this bit flipping should probably be moved to tie-break in the PQ comparator
- longs = new NumericDocValues() {
+ longs = new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) {
@Override
- public long get(int docID) {
- long bits = doubles.get(docID);
+ public long longValue() {
+ long bits = super.longValue();
if (bits < 0) bits ^= 0x7fffffffffffffffL;
return bits;
}
@@ -224,11 +222,13 @@ public class IntervalFacets implements Iterable<FacetInterval> {
default:
throw new AssertionError();
}
- docsWithField = DocValues.getDocsWithField(ctx.reader(), schemaField.getName());
}
- long v = longs.get(doc - ctx.docBase);
- if (v != 0 || docsWithField.get(doc - ctx.docBase)) {
- accumIntervalWithValue(v);
+ int valuesDocID = longs.docID();
+ if (valuesDocID < doc - ctx.docBase) {
+ valuesDocID = longs.advance(doc - ctx.docBase);
+ }
+ if (valuesDocID == doc - ctx.docBase) {
+ accumIntervalWithValue(longs.longValue());
}
}
}
@@ -279,14 +279,17 @@ public class IntervalFacets implements Iterable<FacetInterval> {
if (bits != null && bits.get(doc) == false) {
continue;
}
- ssdv.setDocument(doc);
- long currOrd;
- int currentInterval = 0;
- while ((currOrd = ssdv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- boolean evaluateNextInterval = true;
- while (evaluateNextInterval && currentInterval < intervals.length) {
- IntervalCompareResult result = intervals[currentInterval].includes(currOrd);
- switch (result) {
+ if (doc > ssdv.docID()) {
+ ssdv.advance(doc);
+ }
+ if (doc == ssdv.docID()) {
+ long currOrd;
+ int currentInterval = 0;
+ while ((currOrd = ssdv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ boolean evaluateNextInterval = true;
+ while (evaluateNextInterval && currentInterval < intervals.length) {
+ IntervalCompareResult result = intervals[currentInterval].includes(currOrd);
+ switch (result) {
case INCLUDED:
/*
* Increment the current interval and move to the next one using
@@ -308,6 +311,7 @@ public class IntervalFacets implements Iterable<FacetInterval> {
*/
currentInterval++;
break;
+ }
}
}
}
@@ -324,9 +328,11 @@ public class IntervalFacets implements Iterable<FacetInterval> {
if (bits != null && bits.get(doc) == false) {
continue;
}
- int ord = sdv.getOrd(doc);
- if (ord >= 0) {
- accumInterval(ord);
+ if (doc > sdv.docID()) {
+ sdv.advance(doc);
+ }
+ if (doc == sdv.docID()) {
+ accumInterval(sdv.ordValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/request/NumericFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/NumericFacets.java b/solr/core/src/java/org/apache/solr/request/NumericFacets.java
index fd85d3f..9df0ca6 100644
--- a/solr/core/src/java/org/apache/solr/request/NumericFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/NumericFacets.java
@@ -27,8 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FilterNumericDocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Terms;
@@ -161,25 +162,23 @@ final class NumericFacets {
longs = DocValues.getNumeric(ctx.reader(), fieldName);
break;
case FLOAT:
- final NumericDocValues floats = DocValues.getNumeric(ctx.reader(), fieldName);
// TODO: this bit flipping should probably be moved to tie-break in the PQ comparator
- longs = new NumericDocValues() {
+ longs = new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) {
@Override
- public long get(int docID) {
- long bits = floats.get(docID);
- if (bits<0) bits ^= 0x7fffffffffffffffL;
+ public long longValue() {
+ long bits = super.longValue();
+ if (bits < 0) bits ^= 0x7fffffffffffffffL;
return bits;
}
};
break;
case DOUBLE:
- final NumericDocValues doubles = DocValues.getNumeric(ctx.reader(), fieldName);
// TODO: this bit flipping should probably be moved to tie-break in the PQ comparator
- longs = new NumericDocValues() {
+ longs = new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) {
@Override
- public long get(int docID) {
- long bits = doubles.get(docID);
- if (bits<0) bits ^= 0x7fffffffffffffffL;
+ public long longValue() {
+ long bits = super.longValue();
+ if (bits < 0) bits ^= 0x7fffffffffffffffL;
return bits;
}
};
@@ -187,11 +186,13 @@ final class NumericFacets {
default:
throw new AssertionError();
}
- docsWithField = DocValues.getDocsWithField(ctx.reader(), fieldName);
}
- long v = longs.get(doc - ctx.docBase);
- if (v != 0 || docsWithField.get(doc - ctx.docBase)) {
- hashTable.add(doc, v, 1);
+ int valuesDocID = longs.docID();
+ if (valuesDocID < doc - ctx.docBase) {
+ valuesDocID = longs.advance(doc - ctx.docBase);
+ }
+ if (valuesDocID == doc - ctx.docBase) {
+ hashTable.add(doc, longs.longValue(), 1);
} else {
++missingCount;
}
[14/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsProducer.java
index c73e770..8be0f66 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene53/Lucene53NormsProducer.java
@@ -17,9 +17,6 @@
package org.apache.lucene.codecs.lucene53;
-import static org.apache.lucene.codecs.lucene53.Lucene53NormsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene53.Lucene53NormsFormat.VERSION_START;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -30,13 +27,19 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.LegacyNumericDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.LegacyNumericDocValuesWrapper;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
+import static org.apache.lucene.codecs.lucene53.Lucene53NormsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene53.Lucene53NormsFormat.VERSION_START;
+
/**
* Reader for {@link Lucene53NormsFormat}
*/
@@ -114,22 +117,23 @@ class Lucene53NormsProducer extends NormsProducer {
public NumericDocValues getNorms(FieldInfo field) throws IOException {
final NormsEntry entry = norms.get(field.number);
+ LegacyNumericDocValues norms;
+
if (entry.bytesPerValue == 0) {
final long value = entry.offset;
- return new NumericDocValues() {
- @Override
- public long get(int docID) {
- return value;
- }
- };
- }
-
- RandomAccessInput slice;
- synchronized (data) {
- switch (entry.bytesPerValue) {
+ norms = new LegacyNumericDocValues() {
+ @Override
+ public long get(int docID) {
+ return value;
+ }
+ };
+ } else {
+ RandomAccessInput slice;
+ synchronized (data) {
+ switch (entry.bytesPerValue) {
case 1:
slice = data.randomAccessSlice(entry.offset, maxDoc);
- return new NumericDocValues() {
+ norms = new LegacyNumericDocValues() {
@Override
public long get(int docID) {
try {
@@ -139,9 +143,10 @@ class Lucene53NormsProducer extends NormsProducer {
}
}
};
+ break;
case 2:
slice = data.randomAccessSlice(entry.offset, maxDoc * 2L);
- return new NumericDocValues() {
+ norms = new LegacyNumericDocValues() {
@Override
public long get(int docID) {
try {
@@ -151,9 +156,10 @@ class Lucene53NormsProducer extends NormsProducer {
}
}
};
+ break;
case 4:
slice = data.randomAccessSlice(entry.offset, maxDoc * 4L);
- return new NumericDocValues() {
+ norms = new LegacyNumericDocValues() {
@Override
public long get(int docID) {
try {
@@ -163,9 +169,10 @@ class Lucene53NormsProducer extends NormsProducer {
}
}
};
+ break;
case 8:
slice = data.randomAccessSlice(entry.offset, maxDoc * 8L);
- return new NumericDocValues() {
+ norms = new LegacyNumericDocValues() {
@Override
public long get(int docID) {
try {
@@ -175,10 +182,14 @@ class Lucene53NormsProducer extends NormsProducer {
}
}
};
+ break;
default:
throw new AssertionError();
+ }
}
}
+
+ return new LegacyNumericDocValuesWrapper(new Bits.MatchAllBits(maxDoc), norms);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java
index 96acfd2..217cd43 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java
@@ -32,6 +32,8 @@ import java.util.stream.StreamSupport;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.LegacyDocValuesIterables;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -42,8 +44,8 @@ import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.MathUtil;
-import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
+import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.packed.DirectMonotonicWriter;
import org.apache.lucene.util.packed.DirectWriter;
@@ -85,8 +87,8 @@ final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Close
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- addNumericField(field, values, NumberType.VALUE);
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addNumericField(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, maxDoc), NumberType.VALUE);
}
void addNumericField(FieldInfo field, Iterable<Number> values, NumberType numberType) throws IOException {
@@ -351,7 +353,11 @@ final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Close
}
@Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ addBinaryField(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, maxDoc));
+ }
+
+ private void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
// write the byte[] data
meta.writeVInt(field.number);
meta.writeByte(Lucene54DocValuesFormat.BINARY);
@@ -508,7 +514,6 @@ final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Close
addReverseTermIndex(field, values, maxLength);
}
}
-
// writes term dictionary "block"
// first term is absolute encoded as vint length + bytes.
// lengths of subsequent N terms are encoded as either N bytes or N shorts.
@@ -572,21 +577,32 @@ final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Close
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ meta.writeVInt(field.number);
+ meta.writeByte(Lucene54DocValuesFormat.SORTED);
+ addTermsDict(field, LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field)));
+ addNumericField(field, LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, maxDoc), NumberType.ORDINAL);
+ }
+
+ private void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> ords) throws IOException {
meta.writeVInt(field.number);
meta.writeByte(Lucene54DocValuesFormat.SORTED);
addTermsDict(field, values);
- addNumericField(field, docToOrd, NumberType.ORDINAL);
+ addNumericField(field, ords, NumberType.ORDINAL);
}
@Override
- public void addSortedNumericField(FieldInfo field, final Iterable<Number> docToValueCount, final Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
+
+ final Iterable<Number> docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, maxDoc);
+ final Iterable<Number> values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field);
+
meta.writeVInt(field.number);
meta.writeByte(Lucene54DocValuesFormat.SORTED_NUMERIC);
if (isSingleValued(docToValueCount)) {
meta.writeVInt(SORTED_SINGLE_VALUED);
// The field is single-valued, we can encode it as NUMERIC
- addNumericField(field, singletonView(docToValueCount, values, null));
+ addNumericField(field, singletonView(docToValueCount, values, null), NumberType.VALUE);
} else {
final SortedSet<LongsRef> uniqueValueSets = uniqueValueSets(docToValueCount, values);
if (uniqueValueSets != null) {
@@ -608,7 +624,12 @@ final class Lucene54DocValuesConsumer extends DocValuesConsumer implements Close
}
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+
+ Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field));
+ Iterable<Number> docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, maxDoc);
+ Iterable<Number> ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field);
+
meta.writeVInt(field.number);
meta.writeByte(Lucene54DocValuesFormat.SORTED_SET);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesProducer.java
index 67027ea..62e631c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesProducer.java
@@ -29,20 +29,8 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.NumberType;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
@@ -452,7 +440,59 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.name);
- return getNumeric(entry);
+ Bits docsWithField;
+
+ if (entry.format == SPARSE_COMPRESSED) {
+ // TODO: make a real iterator in this case!
+ docsWithField = getSparseLiveBits(entry);
+ } else {
+ if (entry.missingOffset == ALL_MISSING) {
+ return DocValues.emptyNumeric();
+ } else if (entry.missingOffset == ALL_LIVE) {
+ LongValues values = getNumeric(entry);
+ return new NumericDocValues() {
+ private int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = target;
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+
+ @Override
+ public long longValue() {
+ return values.get(docID);
+ }
+ };
+ } else {
+ docsWithField = getLiveBits(entry.missingOffset, maxDoc);
+ }
+ }
+ return new LegacyNumericDocValuesWrapper(docsWithField, getNumeric(entry));
}
@Override
@@ -545,7 +585,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
}
- static class SparseBits implements Bits {
+ static final class SparseBits implements Bits {
final long maxDoc, docIDsLength, firstDocId;
final LongValues docIds;
@@ -681,8 +721,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
- @Override
- public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ LegacyBinaryDocValues getLegacyBinary(FieldInfo field) throws IOException {
BinaryEntry bytes = binaries.get(field.name);
switch(bytes.format) {
case BINARY_FIXED_UNCOMPRESSED:
@@ -696,7 +735,13 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
}
- private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ @Override
+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ BinaryEntry be = binaries.get(field.name);
+ return new LegacyBinaryDocValuesWrapper(getLiveBits(be.missingOffset, maxDoc), getLegacyBinary(field));
+ }
+
+ private LegacyBinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
final IndexInput data = this.data.slice("fixed-binary", bytes.offset, bytes.count * bytes.maxLength);
final BytesRef term = new BytesRef(bytes.maxLength);
@@ -717,7 +762,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
};
}
- private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ private LegacyBinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
final RandomAccessInput addressesData = this.data.randomAccessSlice(bytes.addressesOffset, bytes.addressesEndOffset - bytes.addressesOffset);
final LongValues addresses = DirectMonotonicReader.getInstance(bytes.addressesMeta, addressesData);
@@ -778,7 +823,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
return index;
}
- private BinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ private LegacyBinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
final MonotonicBlockPackedReader addresses = getIntervalInstance(field, bytes);
final ReverseTermsIndex index = getReverseIndexInstance(field, bytes);
assert addresses.size() > 0; // we don't have to handle empty case
@@ -789,14 +834,55 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
final int valueCount = (int) binaries.get(field.name).count;
- final BinaryDocValues binary = getBinary(field);
+ final LegacyBinaryDocValues binary = getLegacyBinary(field);
NumericEntry entry = ords.get(field.name);
final LongValues ordinals = getNumeric(entry);
return new SortedDocValues() {
+ private int docID = -1;
+ private int ord;
@Override
- public int getOrd(int docID) {
- return (int) ordinals.get(docID);
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assert docID != NO_MORE_DOCS;
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ ord = (int) ordinals.get(docID);
+ if (ord != -1) {
+ break;
+ }
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target-1;
+ return nextDoc();
+ }
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
}
@Override
@@ -847,7 +933,58 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
} else {
docsWithField = getLiveBits(numericEntry.missingOffset, maxDoc);
}
- return DocValues.singleton(values, docsWithField);
+ return new SortedNumericDocValues() {
+ int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+
+ if (docsWithField.get(docID)) {
+ // TODO: use .nextSetBit here, at least!!
+ break;
+ }
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target-1;
+ return nextDoc();
+ }
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+
+ @Override
+ public int docValueCount() {
+ return 1;
+ }
+
+ @Override
+ public long nextValue() {
+ return values.get(docID);
+ }
+ };
} else if (ss.format == SORTED_WITH_ADDRESSES) {
NumericEntry numericEntry = numerics.get(field.name);
final LongValues values = getNumeric(numericEntry);
@@ -856,22 +993,58 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
return new SortedNumericDocValues() {
long startOffset;
long endOffset;
+ int docID = -1;
+ long upto;
@Override
- public void setDocument(int doc) {
- startOffset = ordIndex.get(doc);
- endOffset = ordIndex.get(doc+1L);
+ public int docID() {
+ return docID;
}
@Override
- public long valueAt(int index) {
- return values.get(startOffset + index);
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ startOffset = ordIndex.get(docID);
+ endOffset = ordIndex.get(docID+1L);
+ if (endOffset > startOffset) {
+ break;
+ }
+ }
+ upto = startOffset;
+ return docID;
}
@Override
- public int count() {
+ public int advance(int target) {
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target-1;
+ return nextDoc();
+ }
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+
+ @Override
+ public int docValueCount() {
return (int) (endOffset - startOffset);
}
+
+ @Override
+ public long nextValue() {
+ return values.get(upto++);
+ }
};
} else if (ss.format == SORTED_SET_TABLE) {
NumericEntry entry = ords.get(field.name);
@@ -882,23 +1055,59 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
return new SortedNumericDocValues() {
int startOffset;
int endOffset;
+ int docID = -1;
+ int upto;
@Override
- public void setDocument(int doc) {
- final int ord = (int) ordinals.get(doc);
- startOffset = offsets[ord];
- endOffset = offsets[ord + 1];
+ public int docID() {
+ return docID;
}
@Override
- public long valueAt(int index) {
- return table[startOffset + index];
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ int ord = (int) ordinals.get(docID);
+ startOffset = offsets[ord];
+ endOffset = offsets[ord+1];
+ if (endOffset > startOffset) {
+ break;
+ }
+ }
+ upto = startOffset;
+ return docID;
}
@Override
- public int count() {
+ public int advance(int target) {
+ if (target >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target-1;
+ return nextDoc();
+ }
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
+ }
+
+ @Override
+ public int docValueCount() {
return endOffset - startOffset;
}
+
+ @Override
+ public long nextValue() {
+ return table[upto++];
+ }
};
} else {
throw new AssertionError();
@@ -910,8 +1119,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
SortedSetEntry ss = sortedSets.get(field.name);
switch (ss.format) {
case SORTED_SINGLE_VALUED:
- final SortedDocValues values = getSorted(field);
- return DocValues.singleton(values);
+ return DocValues.singleton(getSorted(field));
case SORTED_WITH_ADDRESSES:
return getSortedSetWithAddresses(field);
case SORTED_SET_TABLE:
@@ -924,12 +1132,12 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
private SortedSetDocValues getSortedSetWithAddresses(FieldInfo field) throws IOException {
final long valueCount = binaries.get(field.name).count;
// we keep the byte[]s and list of ords on disk, these could be large
- final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
+ final LongBinaryDocValues binary = (LongBinaryDocValues) getLegacyBinary(field);
final LongValues ordinals = getNumeric(ords.get(field.name));
// but the addresses to the ord stream are in RAM
final LongValues ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.name));
- return new RandomAccessOrds() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
long startOffset;
long offset;
long endOffset;
@@ -978,28 +1186,18 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
return super.termsEnum();
}
}
-
- @Override
- public long ordAt(int index) {
- return ordinals.get(startOffset + index);
- }
-
- @Override
- public int cardinality() {
- return (int) (endOffset - startOffset);
- }
- };
+ }, maxDoc);
}
private SortedSetDocValues getSortedSetTable(FieldInfo field, SortedSetEntry ss) throws IOException {
final long valueCount = binaries.get(field.name).count;
- final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
+ final LongBinaryDocValues binary = (LongBinaryDocValues) getLegacyBinary(field);
final LongValues ordinals = getNumeric(ords.get(field.name));
final long[] table = ss.table;
final int[] offsets = ss.tableOffsets;
- return new RandomAccessOrds() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
int offset, startOffset, endOffset;
@@ -1011,11 +1209,6 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
@Override
- public long ordAt(int index) {
- return table[startOffset + index];
- }
-
- @Override
public long nextOrd() {
if (offset == endOffset) {
return NO_MORE_ORDS;
@@ -1025,11 +1218,6 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
@Override
- public int cardinality() {
- return endOffset - startOffset;
- }
-
- @Override
public BytesRef lookupOrd(long ord) {
return binary.get(ord);
}
@@ -1056,8 +1244,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
return super.termsEnum();
}
}
-
- };
+ }, maxDoc);
}
private Bits getLiveBits(final long offset, final int count) throws IOException {
@@ -1093,30 +1280,6 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- switch(field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED_NUMERIC:
- return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- BinaryEntry be = binaries.get(field.name);
- return getLiveBits(be.missingOffset, maxDoc);
- case NUMERIC:
- NumericEntry ne = numerics.get(field.name);
- if (ne.format == SPARSE_COMPRESSED) {
- return getSparseLiveBits(ne);
- } else {
- return getLiveBits(ne.missingOffset, maxDoc);
- }
- default:
- throw new AssertionError();
- }
- }
-
- @Override
public synchronized DocValuesProducer getMergeInstance() throws IOException {
return new Lucene54DocValuesProducer(this);
}
@@ -1191,7 +1354,7 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
}
// internally we compose complex dv (sorted/sortedset) from other ones
- static abstract class LongBinaryDocValues extends BinaryDocValues {
+ static abstract class LongBinaryDocValues extends LegacyBinaryDocValues {
@Override
public final BytesRef get(int docID) {
return get((long)docID);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/lucene54/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/package-info.java
index 5dec06b..2c19222 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/package-info.java
@@ -366,7 +366,7 @@
* <li>In version 4.0, the format of the inverted index became extensible via
* the {@link org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage
* ({@code DocValues}) was introduced. Normalization factors need no longer be a
- * single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
+ * single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
* Terms need not be unicode strings, they can be any byte sequence. Term offsets
* can optionally be indexed into the postings lists. Payloads can be stored in the
* term vectors.</li>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
index baadf1e..6772cad 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
@@ -40,8 +40,6 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/**
@@ -105,28 +103,28 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- getInstance(field).addNumericField(field, values);
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ getInstance(field).addNumericField(field, valuesProducer);
}
@Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- getInstance(field).addBinaryField(field, values);
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ getInstance(field).addBinaryField(field, valuesProducer);
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- getInstance(field).addSortedField(field, values, docToOrd);
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ getInstance(field).addSortedField(field, valuesProducer);
}
@Override
- public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
- getInstance(field).addSortedNumericField(field, docToValueCount, values);
+ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ getInstance(field).addSortedNumericField(field, valuesProducer);
}
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
- getInstance(field).addSortedSetField(field, values, docToOrdCount, ords);
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ getInstance(field).addSortedSetField(field, valuesProducer);
}
private DocValuesConsumer getInstance(FieldInfo field) throws IOException {
@@ -308,12 +306,6 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- DocValuesProducer producer = fields.get(field.name);
- return producer == null ? null : producer.getDocsWithField(field);
- }
-
- @Override
public void close() throws IOException {
IOUtils.close(formats.values());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
index b6a16de..94093b2 100644
--- a/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.document;
-
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.util.BytesRef;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
index d60a4e0..f9da94f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
@@ -14,23 +14,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.index;
+package org.apache.lucene.index;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
/**
- * A per-document byte[]
+ * A per-document numeric value.
*/
-public abstract class BinaryDocValues {
+public abstract class BinaryDocValues extends DocIdSetIterator {
/** Sole constructor. (For invocation by subclass
- * constructors, typically implicit.) */
+ * constructors, typically implicit.) */
protected BinaryDocValues() {}
- /** Lookup the value for document. The returned {@link BytesRef} may be
- * re-used across calls to {@link #get(int)} so make sure to
- * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
- * around. */
- public abstract BytesRef get(int docID);
+ /**
+ * Returns the numeric value for the current document ID.
+ * @return numeric value
+ */
+ public abstract BytesRef binaryValue();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
index 03d9ff3..cbcfec5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.store.DataInput;
@@ -115,64 +113,76 @@ class BinaryDocValuesWriter extends DocValuesWriter {
bytes.freeze(false);
final PackedLongValues lengths = this.lengths.build();
dvConsumer.addBinaryField(fieldInfo,
- new Iterable<BytesRef>() {
+ new EmptyDocValuesProducer() {
@Override
- public Iterator<BytesRef> iterator() {
- return new BytesIterator(maxDoc, lengths);
+ public BinaryDocValues getBinary(FieldInfo fieldInfoIn) {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedBinaryDocValues(maxDoc, lengths);
}
});
}
// iterates over the values we have in ram
- private class BytesIterator implements Iterator<BytesRef> {
+ private class BufferedBinaryDocValues extends BinaryDocValues {
final BytesRefBuilder value = new BytesRefBuilder();
final PackedLongValues.Iterator lengthsIterator;
final DataInput bytesIterator = bytes.getDataInput();
- final int size = (int) lengths.size();
+ final int size;
final int maxDoc;
- int upto;
+ private int docID = -1;
- BytesIterator(int maxDoc, PackedLongValues lengths) {
+ BufferedBinaryDocValues(int maxDoc, PackedLongValues lengths) {
this.maxDoc = maxDoc;
this.lengthsIterator = lengths.iterator();
+ this.size = (int) lengths.size();
}
-
+
@Override
- public boolean hasNext() {
- return upto < maxDoc;
+ public int docID() {
+ return docID;
}
@Override
- public BytesRef next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- final BytesRef v;
- if (upto < size) {
- int length = (int) lengthsIterator.next();
- value.grow(length);
- value.setLength(length);
- try {
- bytesIterator.readBytes(value.bytes(), 0, value.length());
- } catch (IOException ioe) {
- // Should never happen!
- throw new RuntimeException(ioe);
- }
- if (docsWithField.get(upto)) {
- v = value.get();
+ public int nextDoc() throws IOException {
+ if (docID == size-1) {
+ docID = NO_MORE_DOCS;
+ } else {
+ int next = docsWithField.nextSetBit(docID+1);
+ if (next == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
} else {
- v = null;
+
+ int length = 0;
+
+ // skip missing values:
+ while (docID < next) {
+ docID++;
+ length = (int) lengthsIterator.next();
+ assert docID == next || length == 0;
+ }
+ value.grow(length);
+ value.setLength(length);
+ bytesIterator.readBytes(value.bytes(), 0, length);
}
- } else {
- v = null;
}
- upto++;
- return v;
+ return docID;
}
@Override
- public void remove() {
+ public int advance(int target) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long cost() {
+ return docsWithField.cardinality();
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return value.get();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 9dee2d1..27d7c50 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -66,6 +66,8 @@ import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Basic tool and API to check the health of an index and
* write a new segments file that removes reference to
@@ -999,7 +1001,7 @@ public final class CheckIndex implements Closeable {
}
for (FieldInfo info : reader.getFieldInfos()) {
if (info.hasNorms()) {
- checkNumericDocValues(info.name, reader.maxDoc(), normsReader.getNorms(info), new Bits.MatchAllBits(reader.maxDoc()));
+ checkNumericDocValues(info.name, normsReader.getNorms(info));
++status.totFields;
}
}
@@ -2056,34 +2058,36 @@ public final class CheckIndex implements Closeable {
}
return status;
}
-
- private static void checkBinaryDocValues(String fieldName, int maxDoc, BinaryDocValues dv, Bits docsWithField) {
- for (int i = 0; i < maxDoc; i++) {
- final BytesRef term = dv.get(i);
- assert term.isValid();
- if (docsWithField.get(i) == false && term.length > 0) {
- throw new RuntimeException("dv for field: " + fieldName + " is missing but has value=" + term + " for doc: " + i);
- }
+
+ private static void checkBinaryDocValues(String fieldName, int maxDoc, BinaryDocValues bdv) throws IOException {
+ int doc;
+ if (bdv.docID() != -1) {
+ throw new RuntimeException("binary dv iterator for field: " + fieldName + " should start at docID=-1, but got " + bdv.docID());
+ }
+ // TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field
+ // TODO: check advance too
+ while ((doc = bdv.nextDoc()) != NO_MORE_DOCS) {
+ BytesRef value = bdv.binaryValue();
+ value.isValid();
}
}
-
- private static void checkSortedDocValues(String fieldName, int maxDoc, SortedDocValues dv, Bits docsWithField) {
- checkBinaryDocValues(fieldName, maxDoc, dv, docsWithField);
+
+ private static void checkSortedDocValues(String fieldName, int maxDoc, SortedDocValues dv) throws IOException {
+ if (dv.docID() != -1) {
+ throw new RuntimeException("sorted dv iterator for field: " + fieldName + " should start at docID=-1, but got " + dv.docID());
+ }
final int maxOrd = dv.getValueCount()-1;
FixedBitSet seenOrds = new FixedBitSet(dv.getValueCount());
int maxOrd2 = -1;
- for (int i = 0; i < maxDoc; i++) {
- int ord = dv.getOrd(i);
+ int docID;
+ // TODO: check advance too
+ while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
+ int ord = dv.ordValue();
if (ord == -1) {
- if (docsWithField.get(i)) {
- throw new RuntimeException("dv for field: " + fieldName + " has -1 ord but is not marked missing for doc: " + i);
- }
+ throw new RuntimeException("dv for field: " + fieldName + " has -1 ord");
} else if (ord < -1 || ord > maxOrd) {
throw new RuntimeException("ord out of bounds: " + ord);
} else {
- if (!docsWithField.get(i)) {
- throw new RuntimeException("dv for field: " + fieldName + " is missing but has ord=" + ord + " for doc: " + i);
- }
maxOrd2 = Math.max(maxOrd2, ord);
seenOrds.set(ord);
}
@@ -2097,7 +2101,7 @@ public final class CheckIndex implements Closeable {
BytesRef lastValue = null;
for (int i = 0; i <= maxOrd; i++) {
final BytesRef term = dv.lookupOrd(i);
- assert term.isValid();
+ term.isValid();
if (lastValue != null) {
if (term.compareTo(lastValue) <= 0) {
throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + term);
@@ -2107,54 +2111,30 @@ public final class CheckIndex implements Closeable {
}
}
- private static void checkSortedSetDocValues(String fieldName, int maxDoc, SortedSetDocValues dv, Bits docsWithField) {
+ private static void checkSortedSetDocValues(String fieldName, int maxDoc, SortedSetDocValues dv) throws IOException {
final long maxOrd = dv.getValueCount()-1;
LongBitSet seenOrds = new LongBitSet(dv.getValueCount());
long maxOrd2 = -1;
- for (int i = 0; i < maxDoc; i++) {
- dv.setDocument(i);
+ int docID;
+ // TODO: check advance too
+ while ((docID = dv.nextDoc()) != NO_MORE_DOCS) {
long lastOrd = -1;
long ord;
- if (docsWithField.get(i)) {
- int ordCount = 0;
- while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- if (ord <= lastOrd) {
- throw new RuntimeException("ords out of order: " + ord + " <= " + lastOrd + " for doc: " + i);
- }
- if (ord < 0 || ord > maxOrd) {
- throw new RuntimeException("ord out of bounds: " + ord);
- }
- if (dv instanceof RandomAccessOrds) {
- long ord2 = ((RandomAccessOrds)dv).ordAt(ordCount);
- if (ord != ord2) {
- throw new RuntimeException("ordAt(" + ordCount + ") inconsistent, expected=" + ord + ",got=" + ord2 + " for doc: " + i);
- }
- }
- lastOrd = ord;
- maxOrd2 = Math.max(maxOrd2, ord);
- seenOrds.set(ord);
- ordCount++;
- }
- if (ordCount == 0) {
- throw new RuntimeException("dv for field: " + fieldName + " has no ordinals but is not marked missing for doc: " + i);
- }
- if (dv instanceof RandomAccessOrds) {
- long ordCount2 = ((RandomAccessOrds)dv).cardinality();
- if (ordCount != ordCount2) {
- throw new RuntimeException("cardinality inconsistent, expected=" + ordCount + ",got=" + ordCount2 + " for doc: " + i);
- }
+ int ordCount = 0;
+ while ((ord = dv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ if (ord <= lastOrd) {
+ throw new RuntimeException("ords out of order: " + ord + " <= " + lastOrd + " for doc: " + docID);
}
- } else {
- long o = dv.nextOrd();
- if (o != SortedSetDocValues.NO_MORE_ORDS) {
- throw new RuntimeException("dv for field: " + fieldName + " is marked missing but has ord=" + o + " for doc: " + i);
- }
- if (dv instanceof RandomAccessOrds) {
- long ordCount2 = ((RandomAccessOrds)dv).cardinality();
- if (ordCount2 != 0) {
- throw new RuntimeException("dv for field: " + fieldName + " is marked missing but has cardinality " + ordCount2 + " for doc: " + i);
- }
+ if (ord < 0 || ord > maxOrd) {
+ throw new RuntimeException("ord out of bounds: " + ord);
}
+ lastOrd = ord;
+ maxOrd2 = Math.max(maxOrd2, ord);
+ seenOrds.set(ord);
+ ordCount++;
+ }
+ if (ordCount == 0) {
+ throw new RuntimeException("dv for field: " + fieldName + " returned docID=" + docID + " yet has no ordinals");
}
}
if (maxOrd != maxOrd2) {
@@ -2177,66 +2157,65 @@ public final class CheckIndex implements Closeable {
}
}
- private static void checkSortedNumericDocValues(String fieldName, int maxDoc, SortedNumericDocValues ndv, Bits docsWithField) {
- for (int i = 0; i < maxDoc; i++) {
- ndv.setDocument(i);
- int count = ndv.count();
- if (docsWithField.get(i)) {
- if (count == 0) {
- throw new RuntimeException("dv for field: " + fieldName + " is not marked missing but has zero count for doc: " + i);
- }
- long previous = Long.MIN_VALUE;
- for (int j = 0; j < count; j++) {
- long value = ndv.valueAt(j);
- if (value < previous) {
- throw new RuntimeException("values out of order: " + value + " < " + previous + " for doc: " + i);
- }
- previous = value;
- }
- } else {
- if (count != 0) {
- throw new RuntimeException("dv for field: " + fieldName + " is marked missing but has count=" + count + " for doc: " + i);
+ private static void checkSortedNumericDocValues(String fieldName, int maxDoc, SortedNumericDocValues ndv) throws IOException {
+ if (ndv.docID() != -1) {
+ throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID());
+ }
+ // TODO: check advance too
+ while (true) {
+ int docID = ndv.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ int count = ndv.docValueCount();
+ if (count == 0) {
+ throw new RuntimeException("sorted numeric dv for field: " + fieldName + " returned docValueCount=0 for docID=" + docID);
+ }
+ long previous = Long.MIN_VALUE;
+ for (int j = 0; j < count; j++) {
+ long value = ndv.nextValue();
+ if (value < previous) {
+ throw new RuntimeException("values out of order: " + value + " < " + previous + " for doc: " + docID);
}
+ previous = value;
}
}
}
- private static void checkNumericDocValues(String fieldName, int maxDoc, NumericDocValues ndv, Bits docsWithField) {
- for (int i = 0; i < maxDoc; i++) {
- long value = ndv.get(i);
- if (docsWithField.get(i) == false && value != 0) {
- throw new RuntimeException("dv for field: " + fieldName + " is marked missing but has value=" + value + " for doc: " + i);
- }
+ private static void checkNumericDocValues(String fieldName, NumericDocValues ndv) throws IOException {
+ int doc;
+ if (ndv.docID() != -1) {
+ throw new RuntimeException("dv iterator for field: " + fieldName + " should start at docID=-1, but got " + ndv.docID());
+ }
+ // TODO: we could add stats to DVs, e.g. total doc count w/ a value for this field
+ // TODO: check advance too
+ while ((doc = ndv.nextDoc()) != NO_MORE_DOCS) {
+ ndv.longValue();
}
}
private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, int maxDoc, PrintStream infoStream, DocValuesStatus status) throws Exception {
- Bits docsWithField = dvReader.getDocsWithField(fi);
- if (docsWithField == null) {
- throw new RuntimeException(fi.name + " docsWithField does not exist");
- } else if (docsWithField.length() != maxDoc) {
- throw new RuntimeException(fi.name + " docsWithField has incorrect length: " + docsWithField.length() + ",expected: " + maxDoc);
- }
switch(fi.getDocValuesType()) {
case SORTED:
status.totalSortedFields++;
- checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi), docsWithField);
+ checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi));
+ checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi));
break;
case SORTED_NUMERIC:
status.totalSortedNumericFields++;
- checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi), docsWithField);
+ checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi));
break;
case SORTED_SET:
status.totalSortedSetFields++;
- checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi), docsWithField);
+ checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi));
break;
case BINARY:
status.totalBinaryFields++;
- checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi), docsWithField);
+ checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi));
break;
case NUMERIC:
status.totalNumericFields++;
- checkNumericDocValues(fi.name, maxDoc, dvReader.getNumeric(fi), docsWithField);
+ checkNumericDocValues(fi.name, dvReader.getNumeric(fi));
break;
default:
throw new AssertionError();
@@ -2464,8 +2443,9 @@ public final class CheckIndex implements Closeable {
*/
public void exorciseIndex(Status result) throws IOException {
ensureOpen();
- if (result.partial)
+ if (result.partial) {
throw new IllegalArgumentException("can only exorcise an index that was fully checked (this status checked a subset of segments)");
+ }
result.newSegments.changed();
result.newSegments.commit(result.dir);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
index 194acd8e..b9ad73a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
@@ -21,9 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.FieldsProducer;
@@ -33,9 +31,6 @@ import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.apache.lucene.util.IOUtils;
/**
* LeafReader implemented by codec APIs.
@@ -109,20 +104,6 @@ public abstract class CodecReader extends LeafReader implements Accountable {
return getPostingsReader();
}
- final CloseableThreadLocal<Map<String,Object>> docValuesLocal = new CloseableThreadLocal<Map<String,Object>>() {
- @Override
- protected Map<String,Object> initialValue() {
- return new HashMap<>();
- }
- };
-
- final CloseableThreadLocal<Map<String,Bits>> docsWithFieldLocal = new CloseableThreadLocal<Map<String,Bits>>() {
- @Override
- protected Map<String,Bits> initialValue() {
- return new HashMap<>();
- }
- };
-
// returns the FieldInfo that corresponds to the given field and type, or
// null if the field does not exist, or not indexed as the requested
// DovDocValuesType.
@@ -143,48 +124,15 @@ public abstract class CodecReader extends LeafReader implements Accountable {
return fi;
}
-
- @Override
- public final NumericDocValues getNumericDocValues(String field) throws IOException {
- ensureOpen();
- Map<String,Object> dvFields = docValuesLocal.get();
-
- Object previous = dvFields.get(field);
- if (previous != null && previous instanceof NumericDocValues) {
- return (NumericDocValues) previous;
- } else {
- FieldInfo fi = getDVField(field, DocValuesType.NUMERIC);
- if (fi == null) {
- return null;
- }
- NumericDocValues dv = getDocValuesReader().getNumeric(fi);
- dvFields.put(field, dv);
- return dv;
- }
- }
@Override
- public final Bits getDocsWithField(String field) throws IOException {
+ public final NumericDocValues getNumericDocValues(String field) throws IOException {
ensureOpen();
- Map<String,Bits> dvFields = docsWithFieldLocal.get();
-
- Bits previous = dvFields.get(field);
- if (previous != null) {
- return previous;
- } else {
- FieldInfo fi = getFieldInfos().fieldInfo(field);
- if (fi == null) {
- // Field does not exist
- return null;
- }
- if (fi.getDocValuesType() == DocValuesType.NONE) {
- // Field was not indexed with doc values
- return null;
- }
- Bits dv = getDocValuesReader().getDocsWithField(fi);
- dvFields.put(field, dv);
- return dv;
+ FieldInfo fi = getDVField(field, DocValuesType.NUMERIC);
+ if (fi == null) {
+ return null;
}
+ return getDocValuesReader().getNumeric(fi);
}
@Override
@@ -194,105 +142,54 @@ public abstract class CodecReader extends LeafReader implements Accountable {
if (fi == null) {
return null;
}
-
- Map<String,Object> dvFields = docValuesLocal.get();
-
- BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field);
- if (dvs == null) {
- dvs = getDocValuesReader().getBinary(fi);
- dvFields.put(field, dvs);
- }
-
- return dvs;
+ return getDocValuesReader().getBinary(fi);
}
@Override
public final SortedDocValues getSortedDocValues(String field) throws IOException {
ensureOpen();
- Map<String,Object> dvFields = docValuesLocal.get();
-
- Object previous = dvFields.get(field);
- if (previous != null && previous instanceof SortedDocValues) {
- return (SortedDocValues) previous;
- } else {
- FieldInfo fi = getDVField(field, DocValuesType.SORTED);
- if (fi == null) {
- return null;
- }
- SortedDocValues dv = getDocValuesReader().getSorted(fi);
- dvFields.put(field, dv);
- return dv;
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED);
+ if (fi == null) {
+ return null;
}
+ return getDocValuesReader().getSorted(fi);
}
@Override
public final SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
ensureOpen();
- Map<String,Object> dvFields = docValuesLocal.get();
- Object previous = dvFields.get(field);
- if (previous != null && previous instanceof SortedNumericDocValues) {
- return (SortedNumericDocValues) previous;
- } else {
- FieldInfo fi = getDVField(field, DocValuesType.SORTED_NUMERIC);
- if (fi == null) {
- return null;
- }
- SortedNumericDocValues dv = getDocValuesReader().getSortedNumeric(fi);
- dvFields.put(field, dv);
- return dv;
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED_NUMERIC);
+ if (fi == null) {
+ return null;
}
+ return getDocValuesReader().getSortedNumeric(fi);
}
@Override
public final SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
ensureOpen();
- Map<String,Object> dvFields = docValuesLocal.get();
-
- Object previous = dvFields.get(field);
- if (previous != null && previous instanceof SortedSetDocValues) {
- return (SortedSetDocValues) previous;
- } else {
- FieldInfo fi = getDVField(field, DocValuesType.SORTED_SET);
- if (fi == null) {
- return null;
- }
- SortedSetDocValues dv = getDocValuesReader().getSortedSet(fi);
- dvFields.put(field, dv);
- return dv;
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED_SET);
+ if (fi == null) {
+ return null;
}
+ return getDocValuesReader().getSortedSet(fi);
}
- final CloseableThreadLocal<Map<String,NumericDocValues>> normsLocal = new CloseableThreadLocal<Map<String,NumericDocValues>>() {
- @Override
- protected Map<String,NumericDocValues> initialValue() {
- return new HashMap<>();
- }
- };
-
@Override
public final NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
- Map<String,NumericDocValues> normFields = normsLocal.get();
-
- NumericDocValues norms = normFields.get(field);
- if (norms != null) {
- return norms;
- } else {
- FieldInfo fi = getFieldInfos().fieldInfo(field);
- if (fi == null || !fi.hasNorms()) {
- // Field does not exist or does not index norms
- return null;
- }
- norms = getNormsReader().getNorms(fi);
- normFields.put(field, norms);
- return norms;
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ if (fi == null || fi.hasNorms() == false) {
+ // Field does not exist or does not index norms
+ return null;
}
+
+ return getNormsReader().getNorms(fi);
}
@Override
protected void doClose() throws IOException {
- IOUtils.close(docValuesLocal, docsWithFieldLocal, normsLocal);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java b/lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java
index 0fb0e3b..cd4726b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java
@@ -17,6 +17,7 @@
package org.apache.lucene.index;
+import java.io.IOException;
import java.util.List;
import org.apache.lucene.search.DocIdSetIterator; // javadocs
@@ -53,11 +54,11 @@ public class DocIDMerger<T extends DocIDMerger.Sub> {
}
/** Returns the next document ID from this sub reader, and {@link DocIdSetIterator#NO_MORE_DOCS} when done */
- public abstract int nextDoc();
+ public abstract int nextDoc() throws IOException;
}
/** Construct this from the provided subs, specifying the maximum sub count */
- public DocIDMerger(List<T> subs, int maxCount, boolean indexIsSorted) {
+ public DocIDMerger(List<T> subs, int maxCount, boolean indexIsSorted) throws IOException {
this.subs = subs;
if (indexIsSorted && maxCount > 1) {
@@ -77,12 +78,12 @@ public class DocIDMerger<T extends DocIDMerger.Sub> {
}
/** Construct this from the provided subs */
- public DocIDMerger(List<T> subs, boolean indexIsSorted) {
+ public DocIDMerger(List<T> subs, boolean indexIsSorted) throws IOException {
this(subs, subs.size(), indexIsSorted);
}
/** Reuse API, currently only used by postings during merge */
- public void reset() {
+ public void reset() throws IOException {
if (queue != null) {
// caller may not have fully consumed the queue:
queue.clear();
@@ -116,7 +117,7 @@ public class DocIDMerger<T extends DocIDMerger.Sub> {
}
/** Returns null when done */
- public T next() {
+ public T next() throws IOException {
// Loop until we find a non-deleted document
if (queue != null) {
T top = queue.top();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/DocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValues.java b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
index 4de4238..3377f45 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
/**
@@ -32,25 +31,80 @@ public final class DocValues {
private DocValues() {}
/**
- * An empty BinaryDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
+ * An empty {@link BinaryDocValues} which returns no documents
*/
public static final BinaryDocValues emptyBinary() {
- final BytesRef empty = new BytesRef();
return new BinaryDocValues() {
+ private boolean exhausted = false;
+
@Override
- public BytesRef get(int docID) {
- return empty;
+ public int advance(int target) {
+ assert exhausted == false;
+ assert target >= 0;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return exhausted ? NO_MORE_DOCS : -1;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert exhausted == false;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ assert false;
+ return null;
}
};
}
/**
- * An empty NumericDocValues which returns zero for every document
+ * An empty NumericDocValues which returns no documents
*/
public static final NumericDocValues emptyNumeric() {
return new NumericDocValues() {
+ private boolean exhausted = false;
+
+ @Override
+ public int advance(int target) {
+ assert exhausted == false;
+ assert target >= 0;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return exhausted ? NO_MORE_DOCS : -1;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert exhausted == false;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
@Override
- public long get(int docID) {
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public long longValue() {
+ assert false;
return 0;
}
};
@@ -59,11 +113,63 @@ public final class DocValues {
/**
* An empty SortedDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
*/
+ public static final LegacySortedDocValues emptyLegacySorted() {
+ final BytesRef empty = new BytesRef();
+ return new LegacySortedDocValues() {
+ @Override
+ public int getOrd(int docID) {
+ return -1;
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ return empty;
+ }
+
+ @Override
+ public int getValueCount() {
+ return 0;
+ }
+ };
+ }
+
+ /**
+ * An empty SortedDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
+ */
public static final SortedDocValues emptySorted() {
final BytesRef empty = new BytesRef();
return new SortedDocValues() {
+
+ private boolean exhausted = false;
+
@Override
- public int getOrd(int docID) {
+ public int advance(int target) {
+ assert exhausted == false;
+ assert target >= 0;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return exhausted ? NO_MORE_DOCS : -1;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert exhausted == false;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public int ordValue() {
+ assert false;
return -1;
}
@@ -83,26 +189,109 @@ public final class DocValues {
* An empty SortedNumericDocValues which returns zero values for every document
*/
public static final SortedNumericDocValues emptySortedNumeric(int maxDoc) {
- return singleton(emptyNumeric(), new Bits.MatchNoBits(maxDoc));
+ return new SortedNumericDocValues() {
+
+ private boolean exhausted = false;
+
+ @Override
+ public int advance(int target) {
+ assert exhausted == false;
+ assert target >= 0;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return exhausted ? NO_MORE_DOCS : -1;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert exhausted == false;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public int docValueCount() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public long nextValue() {
+ throw new IllegalStateException();
+ }
+ };
}
/**
- * An empty SortedDocValues which returns {@link SortedSetDocValues#NO_MORE_ORDS} for every document
+ * An empty SortedDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
*/
- public static final RandomAccessOrds emptySortedSet() {
- return singleton(emptySorted());
+ public static final SortedSetDocValues emptySortedSet() {
+ final BytesRef empty = new BytesRef();
+ return new SortedSetDocValues() {
+
+ private boolean exhausted = false;
+
+ @Override
+ public int advance(int target) {
+ assert exhausted == false;
+ assert target >= 0;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return exhausted ? NO_MORE_DOCS : -1;
+ }
+
+ @Override
+ public int nextDoc() {
+ assert exhausted == false;
+ exhausted = true;
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public long nextOrd() {
+ assert false;
+ return NO_MORE_ORDS;
+ }
+
+ @Override
+ public BytesRef lookupOrd(long ord) {
+ return empty;
+ }
+
+ @Override
+ public long getValueCount() {
+ return 0;
+ }
+ };
}
-
+
/**
- * Returns a multi-valued view over the provided SortedDocValues
+ * Returns a multi-valued view over the provided SortedDocValues
*/
- public static RandomAccessOrds singleton(SortedDocValues dv) {
+ public static SortedSetDocValues singleton(SortedDocValues dv) {
return new SingletonSortedSetDocValues(dv);
}
/**
* Returns a single-valued view of the SortedSetDocValues, if it was previously
- * wrapped with {@link #singleton(SortedDocValues)}, or null.
+ * wrapped with {@link #singleton(SortedDocValues)}, or null.
*/
public static SortedDocValues unwrapSingleton(SortedSetDocValues dv) {
if (dv instanceof SingletonSortedSetDocValues) {
@@ -114,8 +303,7 @@ public final class DocValues {
/**
* Returns a single-valued view of the SortedNumericDocValues, if it was previously
- * wrapped with {@link #singleton(NumericDocValues, Bits)}, or null.
- * @see #unwrapSingletonBits(SortedNumericDocValues)
+ * wrapped with {@link #singleton(NumericDocValues)}, or null.
*/
public static NumericDocValues unwrapSingleton(SortedNumericDocValues dv) {
if (dv instanceof SingletonSortedNumericDocValues) {
@@ -125,76 +313,11 @@ public final class DocValues {
}
}
- /**
- * Returns the documents with a value for the SortedNumericDocValues, if it was previously
- * wrapped with {@link #singleton(NumericDocValues, Bits)}, or null.
- */
- public static Bits unwrapSingletonBits(SortedNumericDocValues dv) {
- if (dv instanceof SingletonSortedNumericDocValues) {
- return ((SingletonSortedNumericDocValues)dv).getDocsWithField();
- } else {
- return null;
- }
- }
-
/**
* Returns a multi-valued view over the provided NumericDocValues
*/
- public static SortedNumericDocValues singleton(NumericDocValues dv, Bits docsWithField) {
- return new SingletonSortedNumericDocValues(dv, docsWithField);
- }
-
- /**
- * Returns a Bits representing all documents from <code>dv</code> that have a value.
- */
- public static Bits docsWithValue(final SortedDocValues dv, final int maxDoc) {
- return new Bits() {
- @Override
- public boolean get(int index) {
- return dv.getOrd(index) >= 0;
- }
-
- @Override
- public int length() {
- return maxDoc;
- }
- };
- }
-
- /**
- * Returns a Bits representing all documents from <code>dv</code> that have a value.
- */
- public static Bits docsWithValue(final SortedSetDocValues dv, final int maxDoc) {
- return new Bits() {
- @Override
- public boolean get(int index) {
- dv.setDocument(index);
- return dv.nextOrd() != SortedSetDocValues.NO_MORE_ORDS;
- }
-
- @Override
- public int length() {
- return maxDoc;
- }
- };
- }
-
- /**
- * Returns a Bits representing all documents from <code>dv</code> that have a value.
- */
- public static Bits docsWithValue(final SortedNumericDocValues dv, final int maxDoc) {
- return new Bits() {
- @Override
- public boolean get(int index) {
- dv.setDocument(index);
- return dv.count() != 0;
- }
-
- @Override
- public int length() {
- return maxDoc;
- }
- };
+ public static SortedNumericDocValues singleton(NumericDocValues dv) {
+ return new SingletonSortedNumericDocValues(dv);
}
// some helpers, for transition from fieldcache apis.
@@ -215,7 +338,7 @@ public final class DocValues {
}
/**
- * Returns NumericDocValues for the field, or {@link #emptyNumeric()} if it has none.
+ * Returns NumericDocValues for the field, or {@link #emptyNumeric()} if it has none.
* @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
* @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
* @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#NUMERIC}.
@@ -232,7 +355,7 @@ public final class DocValues {
}
/**
- * Returns BinaryDocValues for the field, or {@link #emptyBinary} if it has none.
+ * Returns BinaryDocValues for the field, or {@link #emptyBinary} if it has none.
* @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
* @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
* @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#BINARY}
@@ -252,7 +375,7 @@ public final class DocValues {
}
/**
- * Returns SortedDocValues for the field, or {@link #emptySorted} if it has none.
+ * Returns SortedDocValues for the field, or {@link #emptySorted} if it has none.
* @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
* @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
* @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#SORTED}.
@@ -269,7 +392,7 @@ public final class DocValues {
}
/**
- * Returns SortedNumericDocValues for the field, or {@link #emptySortedNumeric} if it has none.
+ * Returns SortedNumericDocValues for the field, or {@link #emptySortedNumeric} if it has none.
* @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
* @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
* @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#SORTED_NUMERIC}
@@ -284,8 +407,7 @@ public final class DocValues {
checkField(reader, field, DocValuesType.SORTED_NUMERIC, DocValuesType.NUMERIC);
return emptySortedNumeric(reader.maxDoc());
}
- Bits bits = reader.getDocsWithField(field);
- return singleton(single, bits);
+ return singleton(single);
}
return dv;
}
@@ -306,29 +428,8 @@ public final class DocValues {
checkField(reader, field, DocValuesType.SORTED, DocValuesType.SORTED_SET);
return emptySortedSet();
}
- return singleton(sorted);
+ dv = singleton(sorted);
}
return dv;
}
-
- /**
- * Returns Bits for the field, or {@link Bits} matching nothing if it has none.
- * @return bits instance, or an empty instance if {@code field} does not exist in this reader.
- * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
- * @throws IOException if an I/O error occurs.
- */
- public static Bits getDocsWithField(LeafReader reader, String field) throws IOException {
- Bits dv = reader.getDocsWithField(field);
- if (dv == null) {
- assert DocValuesType.values().length == 6; // we just don't want NONE
- checkField(reader, field, DocValuesType.BINARY,
- DocValuesType.NUMERIC,
- DocValuesType.SORTED,
- DocValuesType.SORTED_NUMERIC,
- DocValuesType.SORTED_SET);
- return new Bits.MatchNoBits(reader.maxDoc());
- } else {
- return dv;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java
new file mode 100644
index 0000000..a4b9049
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+
+/** Abstrast base class implementing a {@link DocValuesProducer} that has no doc values. */
+public abstract class EmptyDocValuesProducer extends DocValuesProducer {
+
+ /** Sole constructor */
+ protected EmptyDocValuesProducer() {
+ }
+
+ @Override
+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SortedSetDocValues getSortedSet(FieldInfo field) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void checkIntegrity() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** Closes this doc values producer. */
+ @Override
+ public void close() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java
new file mode 100644
index 0000000..26eb12b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Delegates all methods to a wrapped {@link BinaryDocValues}.
+ */
+public abstract class FilterBinaryDocValues extends BinaryDocValues {
+
+ /** Wrapped values */
+ protected final BinaryDocValues in;
+
+ /** Sole constructor */
+ protected FilterBinaryDocValues(BinaryDocValues in) {
+ this.in = in;
+ }
+
+ @Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return in.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return in.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return in.binaryValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
index 886c12a..9e9fcb4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
@@ -467,12 +467,6 @@ public abstract class FilterLeafReader extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- ensureOpen();
- return in.getDocsWithField(field);
- }
-
- @Override
public Sort getIndexSort() {
ensureOpen();
return in.getIndexSort();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java
new file mode 100644
index 0000000..b128d22
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * Delegates all methods to a wrapped {@link NumericDocValues}.
+ */
+public abstract class FilterNumericDocValues extends NumericDocValues {
+
+ /** Wrapped values */
+ protected final NumericDocValues in;
+
+ /** Sole constructor */
+ protected FilterNumericDocValues(NumericDocValues in) {
+ this.in = in;
+ }
+
+ @Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return in.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return in.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public long longValue() {
+ return in.longValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
index 44e61e2..fbabaf1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.index;
-
import java.io.IOException;
import org.apache.lucene.index.IndexReader.ReaderClosedListener;
@@ -241,13 +240,13 @@ public abstract class LeafReader extends IndexReader {
}
/** Returns {@link NumericDocValues} for this field, or
- * null if no {@link NumericDocValues} were indexed for
+ * null if no numeric doc values were indexed for
* this field. The returned instance should only be
- * used by a single thread. */
+ * used by a single thread. This will never return null. */
public abstract NumericDocValues getNumericDocValues(String field) throws IOException;
/** Returns {@link BinaryDocValues} for this field, or
- * null if no {@link BinaryDocValues} were indexed for
+ * null if no binary doc values were indexed for
* this field. The returned instance should only be
* used by a single thread. */
public abstract BinaryDocValues getBinaryDocValues(String field) throws IOException;
@@ -270,12 +269,6 @@ public abstract class LeafReader extends IndexReader {
* used by a single thread. */
public abstract SortedSetDocValues getSortedSetDocValues(String field) throws IOException;
- /** Returns a {@link Bits} at the size of <code>reader.maxDoc()</code>,
- * with turned on bits for each docid that does have a value for this field,
- * or null if no DocValues were indexed for this field. The
- * returned instance should only be used by a single thread */
- public abstract Bits getDocsWithField(String field) throws IOException;
-
/** Returns {@link NumericDocValues} representing norms
* for this field, or null if no {@link NumericDocValues}
* were indexed. The returned instance should only be
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValues.java
new file mode 100644
index 0000000..7aeff7f
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/LegacyBinaryDocValues.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A per-document byte[]
+ *
+ * @deprecated Use {@link BinaryDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacyBinaryDocValues {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LegacyBinaryDocValues() {}
+
+ /** Lookup the value for document. The returned {@link BytesRef} may be
+ * re-used across calls to {@link #get(int)} so make sure to
+ * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
+ * around. */
+ public abstract BytesRef get(int docID);
+}
[08/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
index 83e01e4..93edcc0 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
@@ -113,10 +113,10 @@ final class GlobalOrdinalsQuery extends Query {
return Explanation.noMatch("Not a match");
}
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd == -1) {
+ if (values.advance(doc) != doc) {
return Explanation.noMatch("Not a match");
}
+ int segmentOrd = values.ordValue();
BytesRef joinValue = values.lookupOrd(segmentOrd);
int ord;
@@ -170,8 +170,12 @@ final class GlobalOrdinalsQuery extends Query {
@Override
public boolean matches() throws IOException {
- final long segmentOrd = values.getOrd(approximation.docID());
- if (segmentOrd != -1) {
+ int docID = approximation.docID();
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ if (docID == values.docID()) {
+ final long segmentOrd = values.ordValue();
final long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
if (foundOrds.get(globalOrd)) {
return true;
@@ -204,9 +208,12 @@ final class GlobalOrdinalsQuery extends Query {
@Override
public boolean matches() throws IOException {
- final long segmentOrd = values.getOrd(approximation.docID());
- if (segmentOrd != -1) {
- if (foundOrds.get(segmentOrd)) {
+ int docID = approximation.docID();
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ if (docID == values.docID()) {
+ if (foundOrds.get(values.ordValue())) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
index 57a8442..cc58041 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+import java.util.Arrays;
+
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -26,9 +29,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.LongValues;
-import java.io.IOException;
-import java.util.Arrays;
-
abstract class GlobalOrdinalsWithScoreCollector implements Collector {
final String field;
@@ -113,9 +113,11 @@ abstract class GlobalOrdinalsWithScoreCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(docTermOrds.ordValue());
collectedOrds.set(globalOrd);
float existingScore = scores.getScore(globalOrd);
float newScore = scorer.score();
@@ -143,8 +145,11 @@ abstract class GlobalOrdinalsWithScoreCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final int segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ int segmentOrd = docTermOrds.ordValue();
collectedOrds.set(segmentOrd);
float existingScore = scores.getScore(segmentOrd);
float newScore = scorer.score();
@@ -253,9 +258,11 @@ abstract class GlobalOrdinalsWithScoreCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(docTermOrds.ordValue());
collectedOrds.set(globalOrd);
occurrences.increment(globalOrd);
}
@@ -269,8 +276,11 @@ abstract class GlobalOrdinalsWithScoreCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final int segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ int segmentOrd = docTermOrds.ordValue();
collectedOrds.set(segmentOrd);
occurrences.increment(segmentOrd);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
index a192996..0aedf35 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
@@ -16,15 +16,18 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+import java.util.Set;
+
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.FilterWeight;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FilterWeight;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
@@ -33,9 +36,6 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
-import java.io.IOException;
-import java.util.Set;
-
final class GlobalOrdinalsWithScoreQuery extends Query {
private final GlobalOrdinalsWithScoreCollector collector;
@@ -118,11 +118,11 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
if (values == null) {
return Explanation.noMatch("Not a match");
}
-
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd == -1) {
+ if (values.advance(doc) != doc) {
return Explanation.noMatch("Not a match");
}
+
+ int segmentOrd = values.ordValue();
BytesRef joinValue = values.lookupOrd(segmentOrd);
int ord;
@@ -175,8 +175,12 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
@Override
public boolean matches() throws IOException {
- final long segmentOrd = values.getOrd(approximation.docID());
- if (segmentOrd != -1) {
+ int docID = approximation.docID();
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ if (docID == values.docID()) {
+ final long segmentOrd = values.ordValue();
final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(segmentOrd);
if (collector.match(globalOrd)) {
score = collector.score(globalOrd);
@@ -209,8 +213,12 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
@Override
public boolean matches() throws IOException {
- final int segmentOrd = values.getOrd(approximation.docID());
- if (segmentOrd != -1) {
+ int docID = approximation.docID();
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ if (docID == values.docID()) {
+ final int segmentOrd = values.ordValue();
if (collector.match(segmentOrd)) {
score = collector.score(segmentOrd);
return true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
index b7c2a5b..bfc1f9b 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
@@ -102,12 +102,10 @@ public final class JoinUtil {
termsWithScoreCollector = GenericTermsCollector.createCollectorMV(mvFunction, scoreMode);
} else {
Function<BinaryDocValues> svFunction = DocValuesTermsCollector.binaryDocValues(fromField);
- termsWithScoreCollector = GenericTermsCollector.createCollectorSV(svFunction, scoreMode);
+ termsWithScoreCollector = GenericTermsCollector.createCollectorSV(svFunction, scoreMode);
}
- return createJoinQuery(multipleValuesPerDocument, toField, fromQuery, fromSearcher, scoreMode,
- termsWithScoreCollector);
-
+ return createJoinQuery(multipleValuesPerDocument, toField, fromQuery, fromSearcher, scoreMode, termsWithScoreCollector);
}
/**
@@ -202,12 +200,16 @@ public final class JoinUtil {
@Override
public void collect(int doc) throws IOException {
- sortedNumericDocValues.setDocument(doc);
- for (int i = 0; i < sortedNumericDocValues.count(); i++) {
- long value = sortedNumericDocValues.valueAt(i);
- joinValues.add(value);
- if (needsScore) {
- scoreAggregator.accept(value, scorer.score());
+ if (doc > sortedNumericDocValues.docID()) {
+ sortedNumericDocValues.advance(doc);
+ }
+ if (doc == sortedNumericDocValues.docID()) {
+ for (int i = 0; i < sortedNumericDocValues.docValueCount(); i++) {
+ long value = sortedNumericDocValues.nextValue();
+ joinValues.add(value);
+ if (needsScore) {
+ scoreAggregator.accept(value, scorer.score());
+ }
}
}
}
@@ -232,10 +234,29 @@ public final class JoinUtil {
NumericDocValues numericDocValues;
Scorer scorer;
+ private int lastDocID = -1;
+
+ private boolean docsInOrder(int docID) {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ return true;
+ }
@Override
public void collect(int doc) throws IOException {
- long value = numericDocValues.get(doc);
+ assert docsInOrder(doc);
+ int dvDocID = numericDocValues.docID();
+ if (dvDocID < doc) {
+ dvDocID = numericDocValues.advance(doc);
+ }
+ long value;
+ if (dvDocID == doc) {
+ value = numericDocValues.longValue();
+ } else {
+ value = 0;
+ }
joinValues.add(value);
if (needsScore) {
scoreAggregator.accept(value, scorer.score());
@@ -245,6 +266,7 @@ public final class JoinUtil {
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
numericDocValues = DocValues.getNumeric(context.reader(), fromField);
+ lastDocID = -1;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
index f2b3d42..8475af7 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
@@ -40,7 +40,6 @@ abstract class TermsCollector<DV> extends DocValuesTermsCollector<DV> {
return collectorTerms;
}
-
/**
* Chooses the right {@link TermsCollector} implementation.
*
@@ -64,10 +63,14 @@ abstract class TermsCollector<DV> extends DocValuesTermsCollector<DV> {
@Override
public void collect(int doc) throws IOException {
long ord;
- docValues.setDocument(doc);
- while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- final BytesRef term = docValues.lookupOrd(ord);
- collectorTerms.add(term);
+ if (doc > docValues.docID()) {
+ docValues.advance(doc);
+ }
+ if (doc == docValues.docID()) {
+ while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ final BytesRef term = docValues.lookupOrd(ord);
+ collectorTerms.add(term);
+ }
}
}
}
@@ -81,7 +84,15 @@ abstract class TermsCollector<DV> extends DocValuesTermsCollector<DV> {
@Override
public void collect(int doc) throws IOException {
- final BytesRef term = docValues.get(doc);
+ if (docValues.docID() < doc) {
+ docValues.advance(doc);
+ }
+ BytesRef term;
+ if (docValues.docID() == doc) {
+ term = docValues.binaryValue();
+ } else {
+ term = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
collectorTerms.add(term);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
index a11dcec..22fc54d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@@ -95,7 +96,16 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public void collect(int doc) throws IOException {
- int ord = collectedTerms.add(docValues.get(doc));
+ if (docValues.docID() < doc) {
+ docValues.advance(doc);
+ }
+ BytesRef value;
+ if (docValues.docID() == doc) {
+ value = docValues.binaryValue();
+ } else {
+ value = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
+ int ord = collectedTerms.add(value);
if (ord < 0) {
ord = -ord - 1;
} else {
@@ -145,7 +155,16 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public void collect(int doc) throws IOException {
- int ord = collectedTerms.add(docValues.get(doc));
+ if (docValues.docID() < doc) {
+ docValues.advance(doc);
+ }
+ BytesRef value;
+ if (docValues.docID() == doc) {
+ value = docValues.binaryValue();
+ } else {
+ value = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
+ int ord = collectedTerms.add(value);
if (ord < 0) {
ord = -ord - 1;
} else {
@@ -188,25 +207,28 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public void collect(int doc) throws IOException {
- docValues.setDocument(doc);
- long ord;
- while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- int termID = collectedTerms.add(docValues.lookupOrd(ord));
- if (termID < 0) {
- termID = -termID - 1;
- } else {
- if (termID >= scoreSums.length) {
- int begin = scoreSums.length;
- scoreSums = ArrayUtil.grow(scoreSums);
- if (scoreMode == ScoreMode.Min) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.POSITIVE_INFINITY);
- } else if (scoreMode == ScoreMode.Max) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.NEGATIVE_INFINITY);
+ if (doc > docValues.docID()) {
+ docValues.advance(doc);
+ }
+ if (doc == docValues.docID()) {
+ long ord;
+ while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ int termID = collectedTerms.add(docValues.lookupOrd(ord));
+ if (termID < 0) {
+ termID = -termID - 1;
+ } else {
+ if (termID >= scoreSums.length) {
+ int begin = scoreSums.length;
+ scoreSums = ArrayUtil.grow(scoreSums);
+ if (scoreMode == ScoreMode.Min) {
+ Arrays.fill(scoreSums, begin, scoreSums.length, Float.POSITIVE_INFINITY);
+ } else if (scoreMode == ScoreMode.Max) {
+ Arrays.fill(scoreSums, begin, scoreSums.length, Float.NEGATIVE_INFINITY);
+ }
}
}
- }
- switch (scoreMode) {
+ switch (scoreMode) {
case Total:
scoreSums[termID] += scorer.score();
break;
@@ -218,6 +240,7 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
break;
default:
throw new AssertionError("unexpected: " + scoreMode);
+ }
}
}
}
@@ -232,21 +255,25 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public void collect(int doc) throws IOException {
- docValues.setDocument(doc);
- long ord;
- while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- int termID = collectedTerms.add(docValues.lookupOrd(ord));
- if (termID < 0) {
- termID = -termID - 1;
- } else {
- if (termID >= scoreSums.length) {
- scoreSums = ArrayUtil.grow(scoreSums);
- scoreCounts = ArrayUtil.grow(scoreCounts);
+ if (doc > docValues.docID()) {
+ docValues.advance(doc);
+ }
+ if (doc == docValues.docID()) {
+ long ord;
+ while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ int termID = collectedTerms.add(docValues.lookupOrd(ord));
+ if (termID < 0) {
+ termID = -termID - 1;
+ } else {
+ if (termID >= scoreSums.length) {
+ scoreSums = ArrayUtil.grow(scoreSums);
+ scoreCounts = ArrayUtil.grow(scoreCounts);
+ }
}
- }
- scoreSums[termID] += scorer.score();
- scoreCounts[termID]++;
+ scoreSums[termID] += scorer.score();
+ scoreCounts[termID]++;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
index d3ff410..0777150 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
@@ -16,7 +16,10 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FilterNumericDocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
@@ -25,11 +28,8 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.NumericUtils;
-import java.io.IOException;
-
/**
* A special sort field that allows sorting parent docs based on nested / child level fields.
* Based on the sort order it either takes the document with the lowest or highest field value into account.
@@ -139,16 +139,6 @@ public class ToParentBlockJoinSortField extends SortField {
}
return BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
}
- @Override
- protected Bits getDocsWithValue(LeafReaderContext context, String field) throws IOException {
- final Bits docsWithValue = DocValues.getDocsWithField(context.reader(), field);
- final BitSet parents = parentFilter.getBitSet(context);
- final BitSet children = childFilter.getBitSet(context);
- if (children == null) {
- return new Bits.MatchNoBits(context.reader().maxDoc());
- }
- return BlockJoinSelector.wrap(docsWithValue, parents, children);
- }
};
}
@@ -167,16 +157,6 @@ public class ToParentBlockJoinSortField extends SortField {
}
return BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
}
- @Override
- protected Bits getDocsWithValue(LeafReaderContext context, String field) throws IOException {
- final Bits docsWithValue = DocValues.getDocsWithField(context.reader(), field);
- final BitSet parents = parentFilter.getBitSet(context);
- final BitSet children = childFilter.getBitSet(context);
- if (children == null) {
- return new Bits.MatchNoBits(context.reader().maxDoc());
- }
- return BlockJoinSelector.wrap(docsWithValue, parents, children);
- }
};
}
@@ -193,12 +173,11 @@ public class ToParentBlockJoinSortField extends SortField {
if (children == null) {
return DocValues.emptyNumeric();
}
- final NumericDocValues view = BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
- // undo the numericutils sortability
- return new NumericDocValues() {
+ return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, children)) {
@Override
- public long get(int docID) {
- return NumericUtils.sortableFloatBits((int) view.get(docID));
+ public long longValue() {
+ // undo the numericutils sortability
+ return NumericUtils.sortableFloatBits((int) super.longValue());
}
};
}
@@ -218,25 +197,14 @@ public class ToParentBlockJoinSortField extends SortField {
if (children == null) {
return DocValues.emptyNumeric();
}
- final NumericDocValues view = BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
- // undo the numericutils sortability
- return new NumericDocValues() {
+ return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, children)) {
@Override
- public long get(int docID) {
- return NumericUtils.sortableDoubleBits(view.get(docID));
+ public long longValue() {
+ // undo the numericutils sortability
+ return NumericUtils.sortableDoubleBits(super.longValue());
}
};
}
- @Override
- protected Bits getDocsWithValue(LeafReaderContext context, String field) throws IOException {
- final Bits docsWithValue = DocValues.getDocsWithField(context.reader(), field);
- final BitSet parents = parentFilter.getBitSet(context);
- final BitSet children = childFilter.getBitSet(context);
- if (children == null) {
- return new Bits.MatchNoBits(context.reader().maxDoc());
- }
- return BlockJoinSelector.wrap(docsWithValue, parents, children);
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
index c4cdc3b..41f994c 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.index.DocValues;
@@ -27,6 +28,8 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestBlockJoinSelector extends LuceneTestCase {
public void testDocsWithValue() {
@@ -61,7 +64,7 @@ public class TestBlockJoinSelector extends LuceneTestCase {
assertFalse(docsWithValue.get(19));
}
- public void testSortedSelector() {
+ public void testSortedSelector() throws IOException {
final BitSet parents = new FixedBitSet(20);
parents.set(0);
parents.set(5);
@@ -84,43 +87,91 @@ public class TestBlockJoinSelector extends LuceneTestCase {
ords[4] = 3;
ords[12] = 10;
ords[18] = 10;
- final SortedDocValues values = new SortedDocValues() {
- @Override
- public int getOrd(int docID) {
- return ords[docID];
- }
+ final SortedDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MIN, parents, children);
+ assertEquals(5, mins.nextDoc());
+ assertEquals(3, mins.ordValue());
+ assertEquals(15, mins.nextDoc());
+ assertEquals(10, mins.ordValue());
+ assertEquals(19, mins.nextDoc());
+ assertEquals(10, mins.ordValue());
+ assertEquals(NO_MORE_DOCS, mins.nextDoc());
+
+ final SortedDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MAX, parents, children);
+ assertEquals(5, maxs.nextDoc());
+ assertEquals(7, maxs.ordValue());
+ assertEquals(15, maxs.nextDoc());
+ assertEquals(10, maxs.ordValue());
+ assertEquals(19, maxs.nextDoc());
+ assertEquals(10, maxs.ordValue());
+ assertEquals(NO_MORE_DOCS, maxs.nextDoc());
+ }
+
+ private static class CannedSortedDocValues extends SortedDocValues {
+ private final int[] ords;
+ int docID = -1;
- @Override
- public BytesRef lookupOrd(int ord) {
- throw new UnsupportedOperationException();
+ public CannedSortedDocValues(int[] ords) {
+ this.ords = ords;
+ }
+
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == ords.length) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ if (ords[docID] != -1) {
+ break;
+ }
}
+ return docID;
+ }
- @Override
- public int getValueCount() {
- return 11;
+ @Override
+ public int advance(int target) {
+ if (target >= ords.length) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = target;
+ if (ords[docID] == -1) {
+ nextDoc();
+ }
}
+ return docID;
+ }
+
+ @Override
+ public int ordValue() {
+ assert ords[docID] != -1;
+ return ords[docID];
+ }
+
+ @Override
+ public long cost() {
+ return 5;
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ throw new UnsupportedOperationException();
+ }
- };
-
- final SortedDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(values), BlockJoinSelector.Type.MIN, parents, children);
- assertEquals(-1, mins.getOrd(0));
- assertEquals(3, mins.getOrd(5));
- assertEquals(-1, mins.getOrd(6));
- assertEquals(-1, mins.getOrd(10));
- assertEquals(10, mins.getOrd(15));
- assertEquals(-1, mins.getOrd(19));
-
- final SortedDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(values), BlockJoinSelector.Type.MAX, parents, children);
- assertEquals(-1, maxs.getOrd(0));
- assertEquals(7, maxs.getOrd(5));
- assertEquals(-1, maxs.getOrd(6));
- assertEquals(-1, maxs.getOrd(10));
- assertEquals(10, maxs.getOrd(15));
- assertEquals(-1, maxs.getOrd(19));
+ @Override
+ public int getValueCount() {
+ return 11;
+ }
}
- public void testNumericSelector() {
+ public void testNumericSelector() throws Exception {
final BitSet parents = new FixedBitSet(20);
parents.set(0);
parents.set(5);
@@ -148,29 +199,71 @@ public class TestBlockJoinSelector extends LuceneTestCase {
longs[12] = 10;
docsWithValue.set(18);
longs[18] = 10;
- final NumericDocValues values = new NumericDocValues() {
- @Override
- public long get(int docID) {
- return longs[docID];
+ final NumericDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MIN, parents, children);
+ assertEquals(5, mins.nextDoc());
+ assertEquals(3, mins.longValue());
+ assertEquals(15, mins.nextDoc());
+ assertEquals(10, mins.longValue());
+ assertEquals(NO_MORE_DOCS, mins.nextDoc());
+
+ final NumericDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MAX, parents, children);
+ assertEquals(5, maxs.nextDoc());
+ assertEquals(7, maxs.longValue());
+ assertEquals(15, maxs.nextDoc());
+ assertEquals(10, maxs.longValue());
+ assertEquals(NO_MORE_DOCS, maxs.nextDoc());
+ }
+
+ private static class CannedNumericDocValues extends NumericDocValues {
+ final Bits docsWithValue;
+ final long[] values;
+ int docID = -1;
+
+ public CannedNumericDocValues(long[] values, Bits docsWithValue) {
+ this.values = values;
+ this.docsWithValue = docsWithValue;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == values.length) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ if (docsWithValue.get(docID)) {
+ break;
+ }
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= values.length) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target - 1;
+ return nextDoc();
}
-
- };
-
- final NumericDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(values, docsWithValue), BlockJoinSelector.Type.MIN, parents, children);
- assertEquals(0, mins.get(0));
- assertEquals(3, mins.get(5));
- assertEquals(0, mins.get(6));
- assertEquals(0, mins.get(10));
- assertEquals(10, mins.get(15));
- assertEquals(0, mins.get(19));
-
- final NumericDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(values, docsWithValue), BlockJoinSelector.Type.MAX, parents, children);
- assertEquals(0, maxs.get(0));
- assertEquals(7, maxs.get(5));
- assertEquals(0, maxs.get(6));
- assertEquals(0, maxs.get(10));
- assertEquals(10, maxs.get(15));
- assertEquals(0, maxs.get(19));
+ }
+
+ @Override
+ public long longValue() {
+ return values[docID];
+ }
+
+ @Override
+ public long cost() {
+ return 5;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java
index 8b2a0bd..0d53c85 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java
@@ -16,13 +16,15 @@
*/
package org.apache.lucene.search.join;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.FieldDoc;
@@ -37,9 +39,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.List;
-
/**
*/
public class TestBlockJoinSorting extends LuceneTestCase {
@@ -48,7 +47,7 @@ public class TestBlockJoinSorting extends LuceneTestCase {
public void testNestedSorting() throws Exception {
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setMergePolicy(NoMergePolicy.INSTANCE));
+ .setMergePolicy(newLogMergePolicy()));
List<Document> docs = new ArrayList<>();
Document document = new Document();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 17cd4a2..2bd8381 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -54,8 +54,8 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.NumericDocValues;
@@ -88,7 +88,6 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -484,7 +483,8 @@ public class TestJoinUtil extends LuceneTestCase {
return new FilterScorer(fieldScorer, this) {
@Override
public float score() throws IOException {
- return (float) price.get(in.docID());
+ assertEquals(in.docID(), price.nextDoc());
+ return (float) price.longValue();
}
};
}
@@ -1182,15 +1182,19 @@ public class TestJoinUtil extends LuceneTestCase {
@Override
public void collect(int doc) throws IOException {
- docTermOrds.setDocument(doc);
- long ord;
- while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- final BytesRef joinValue = docTermOrds.lookupOrd(ord);
- JoinScore joinScore = joinValueToJoinScores.get(joinValue);
- if (joinScore == null) {
- joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ long ord;
+ while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ final BytesRef joinValue = docTermOrds.lookupOrd(ord);
+ JoinScore joinScore = joinValueToJoinScores.get(joinValue);
+ if (joinScore == null) {
+ joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
+ }
+ joinScore.addScore(scorer.score());
}
- joinScore.addScore(scorer.score());
}
}
@@ -1214,12 +1218,17 @@ public class TestJoinUtil extends LuceneTestCase {
private Scorer scorer;
private BinaryDocValues terms;
- private Bits docsWithField;
@Override
public void collect(int doc) throws IOException {
- final BytesRef joinValue = terms.get(doc);
- if (joinValue.length == 0 && !docsWithField.get(doc)) {
+ if (doc > terms.docID()) {
+ terms.advance(doc);
+ }
+ final BytesRef joinValue;
+ if (doc == terms.docID()) {
+ joinValue = terms.binaryValue();
+ } else {
+ // missing;
return;
}
@@ -1236,7 +1245,6 @@ public class TestJoinUtil extends LuceneTestCase {
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
terms = DocValues.getBinary(context.reader(), fromField);
- docsWithField = DocValues.getDocsWithField(context.reader(), fromField);
}
@Override
@@ -1281,8 +1289,17 @@ public class TestJoinUtil extends LuceneTestCase {
private int docBase;
@Override
- public void collect(int doc) {
- final BytesRef joinValue = terms.get(doc);
+ public void collect(int doc) throws IOException {
+ if (doc > terms.docID()) {
+ terms.advance(doc);
+ }
+ final BytesRef joinValue;
+ if (doc == terms.docID()) {
+ joinValue = terms.binaryValue();
+ } else {
+ // missing;
+ joinValue = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
JoinScore joinScore = joinValueToJoinScores.get(joinValue);
if (joinScore == null) {
return;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 9037dfa..8db9c9e 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -828,7 +828,7 @@ public class MemoryIndex {
private FieldInfo fieldInfo;
/** The norms for this field; computed on demand. */
- private transient NumericDocValues norms;
+ private transient LegacyNumericDocValues norms;
/**
* Term strings and their positions for this field: Map <String
@@ -937,24 +937,47 @@ public class MemoryIndex {
}
NumericDocValues getNormDocValues() {
- if (norms == null) {
- FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
- numTokens, numOverlapTokens, 0, boost);
- final long value = normSimilarity.computeNorm(invertState);
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldInfo.name + ":" + value + ":" + numTokens);
- norms = new NumericDocValues() {
+ FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
+ numTokens, numOverlapTokens, 0, boost);
+ final long value = normSimilarity.computeNorm(invertState);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldInfo.name + ":" + value + ":" + numTokens);
+ return new NumericDocValues() {
+ private int docID = -1;
@Override
- public long get(int docID) {
- if (docID != 0)
- throw new IndexOutOfBoundsException();
- else
- return value;
+ public int nextDoc() {
+ docID++;
+ if (docID == 1) {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
}
+ @Override
+ public int advance(int target) {
+ if (docID <= 0 && target == 0) {
+ docID = 0;
+ } else {
+ docID = NO_MORE_DOCS;
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return 1;
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
};
- }
- return norms;
}
}
@@ -965,13 +988,13 @@ public class MemoryIndex {
private static final class BinaryDocValuesProducer {
BytesRefHash dvBytesValuesSet;
- final SortedDocValues sortedDocValues;
+ final LegacySortedDocValues sortedDocValues;
final BytesRef spare = new BytesRef();
int[] bytesIds;
private BinaryDocValuesProducer() {
- sortedDocValues = new SortedDocValues() {
+ sortedDocValues = new LegacySortedDocValues() {
@Override
public int getOrd(int docID) {
return 0;
@@ -1004,17 +1027,17 @@ public class MemoryIndex {
long[] dvLongValues;
int count;
- final NumericDocValues numericDocValues;
- final SortedNumericDocValues sortedNumericDocValues;
+ final LegacyNumericDocValues numericDocValues;
+ final LegacySortedNumericDocValues sortedNumericDocValues;
private NumericDocValuesProducer() {
- this.numericDocValues = new NumericDocValues() {
+ this.numericDocValues = new LegacyNumericDocValues() {
@Override
public long get(int docID) {
return dvLongValues[0];
}
};
- this.sortedNumericDocValues = new SortedNumericDocValues() {
+ this.sortedNumericDocValues = new LegacySortedNumericDocValues() {
@Override
public void setDocument(int doc) {
}
@@ -1101,13 +1124,12 @@ public class MemoryIndex {
}
@Override
- public NumericDocValues getNumericDocValues(String field) {
+ public NumericDocValues getNumericDocValues(String field) throws IOException {
Info info = getInfoForExpectedDocValuesType(field, DocValuesType.NUMERIC);
- if (info != null) {
- return info.numericProducer.numericDocValues;
- } else {
+ if (info == null) {
return null;
}
+ return new LegacyNumericDocValuesWrapper(new Bits.MatchAllBits(1), info.numericProducer.numericDocValues);
}
@Override
@@ -1123,7 +1145,7 @@ public class MemoryIndex {
private SortedDocValues getSortedDocValues(String field, DocValuesType docValuesType) {
Info info = getInfoForExpectedDocValuesType(field, docValuesType);
if (info != null) {
- return info.binaryProducer.sortedDocValues;
+ return new LegacySortedDocValuesWrapper(info.binaryProducer.sortedDocValues, 1);
} else {
return null;
}
@@ -1133,7 +1155,7 @@ public class MemoryIndex {
public SortedNumericDocValues getSortedNumericDocValues(String field) {
Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_NUMERIC);
if (info != null) {
- return info.numericProducer.sortedNumericDocValues;
+ return new LegacySortedNumericDocValuesWrapper(info.numericProducer.sortedNumericDocValues, 1);
} else {
return null;
}
@@ -1143,7 +1165,7 @@ public class MemoryIndex {
public SortedSetDocValues getSortedSetDocValues(String field) {
Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_SET);
if (info != null) {
- return new SortedSetDocValues() {
+ return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
int index = 0;
@@ -1169,17 +1191,7 @@ public class MemoryIndex {
public long getValueCount() {
return info.binaryProducer.dvBytesValuesSet.size();
}
- };
- } else {
- return null;
- }
- }
-
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- Info info = fields.get(field);
- if (info != null && info.fieldInfo.getDocValuesType() != DocValuesType.NONE) {
- return new Bits.MatchAllBits(1);
+ }, 1);
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 1d7c60e..da3dd4c 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -180,7 +180,9 @@ public class TestMemoryIndex extends LuceneTestCase {
IndexSearcher searcher = mi.createSearcher();
LeafReader reader = (LeafReader) searcher.getIndexReader();
- float n1 = reader.getNormValues("f1").get(0);
+ NumericDocValues norms = reader.getNormValues("f1");
+ assertEquals(0, norms.nextDoc());
+ float n1 = norms.longValue();
// Norms are re-computed when we change the Similarity
mi.setSimilarity(new ClassicSimilarity() {
@@ -189,7 +191,9 @@ public class TestMemoryIndex extends LuceneTestCase {
return 74;
}
});
- float n2 = reader.getNormValues("f1").get(0);
+ norms = reader.getNormValues("f1");
+ assertEquals(0, norms.nextDoc());
+ float n2 = norms.longValue();
assertTrue(n1 != n2);
TestUtil.checkReader(reader);
@@ -235,24 +239,27 @@ public class TestMemoryIndex extends LuceneTestCase {
MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
- assertEquals(29L, numericDocValues.get(0));
+ assertEquals(0, numericDocValues.nextDoc());
+ assertEquals(29L, numericDocValues.longValue());
SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
- sortedNumericDocValues.setDocument(0);
- assertEquals(5, sortedNumericDocValues.count());
- assertEquals(30L, sortedNumericDocValues.valueAt(0));
- assertEquals(31L, sortedNumericDocValues.valueAt(1));
- assertEquals(32L, sortedNumericDocValues.valueAt(2));
- assertEquals(32L, sortedNumericDocValues.valueAt(3));
- assertEquals(33L, sortedNumericDocValues.valueAt(4));
+ assertEquals(0, sortedNumericDocValues.nextDoc());
+ assertEquals(5, sortedNumericDocValues.docValueCount());
+ assertEquals(30L, sortedNumericDocValues.nextValue());
+ assertEquals(31L, sortedNumericDocValues.nextValue());
+ assertEquals(32L, sortedNumericDocValues.nextValue());
+ assertEquals(32L, sortedNumericDocValues.nextValue());
+ assertEquals(33L, sortedNumericDocValues.nextValue());
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
- assertEquals("a", binaryDocValues.get(0).utf8ToString());
+ assertEquals(0, binaryDocValues.nextDoc());
+ assertEquals("a", binaryDocValues.binaryValue().utf8ToString());
SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
- assertEquals("b", sortedDocValues.get(0).utf8ToString());
- assertEquals(0, sortedDocValues.getOrd(0));
+ assertEquals(0, sortedDocValues.nextDoc());
+ assertEquals("b", sortedDocValues.binaryValue().utf8ToString());
+ assertEquals(0, sortedDocValues.ordValue());
assertEquals("b", sortedDocValues.lookupOrd(0).utf8ToString());
SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
assertEquals(3, sortedSetDocValues.getValueCount());
- sortedSetDocValues.setDocument(0);
+ assertEquals(0, sortedSetDocValues.nextDoc());
assertEquals(0L, sortedSetDocValues.nextOrd());
assertEquals(1L, sortedSetDocValues.nextOrd());
assertEquals(2L, sortedSetDocValues.nextOrd());
@@ -335,7 +342,8 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals(5, penum.endOffset());
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text");
- assertEquals("quick brown fox", binaryDocValues.get(0).utf8ToString());
+ assertEquals(0, binaryDocValues.nextDoc());
+ assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString());
}
public void testPointValues() throws Exception {
@@ -475,7 +483,9 @@ public class TestMemoryIndex extends LuceneTestCase {
assertArrayEquals(packedPoint, leafReader.getPointValues().getMinPackedValue("field"));
assertArrayEquals(packedPoint, leafReader.getPointValues().getMaxPackedValue("field"));
- assertEquals("term", leafReader.getBinaryDocValues("field").get(0).utf8ToString());
+ BinaryDocValues dvs = leafReader.getBinaryDocValues("field");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals("term", dvs.binaryValue().utf8ToString());
}
public void testToStringDebug() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
index a785720..03c17a5 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
@@ -52,26 +52,8 @@ import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CompositeReader;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
@@ -201,7 +183,9 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
if (normValues != null) {
// mem idx always computes norms on the fly
assertNotNull(memNormValues);
- assertEquals(normValues.get(0), memNormValues.get(0));
+ assertEquals(0, normValues.nextDoc());
+ assertEquals(0, memNormValues.nextDoc());
+ assertEquals(normValues.longValue(), memNormValues.longValue());
}
assertNotNull(memTerms);
@@ -503,32 +487,38 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
NumericDocValues controlNumericDocValues = controlLeafReader.getNumericDocValues("numeric");
- assertEquals(controlNumericDocValues.get(0), numericDocValues.get(0));
+ assertEquals(0, numericDocValues.nextDoc());
+ assertEquals(0, controlNumericDocValues.nextDoc());
+ assertEquals(controlNumericDocValues.longValue(), numericDocValues.longValue());
SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
- sortedNumericDocValues.setDocument(0);
+ assertEquals(0, sortedNumericDocValues.nextDoc());
SortedNumericDocValues controlSortedNumericDocValues = controlLeafReader.getSortedNumericDocValues("sorted_numeric");
- controlSortedNumericDocValues.setDocument(0);
- assertEquals(controlSortedNumericDocValues.count(), sortedNumericDocValues.count());
- for (int i = 0; i < controlSortedNumericDocValues.count(); i++) {
- assertEquals(controlSortedNumericDocValues.valueAt(i), sortedNumericDocValues.valueAt(i));
+ assertEquals(0, controlSortedNumericDocValues.nextDoc());
+ assertEquals(controlSortedNumericDocValues.docValueCount(), sortedNumericDocValues.docValueCount());
+ for (int i = 0; i < controlSortedNumericDocValues.docValueCount(); i++) {
+ assertEquals(controlSortedNumericDocValues.nextValue(), sortedNumericDocValues.nextValue());
}
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
BinaryDocValues controlBinaryDocValues = controlLeafReader.getBinaryDocValues("binary");
- assertEquals(controlBinaryDocValues.get(0), binaryDocValues.get(0));
+ assertEquals(0, binaryDocValues.nextDoc());
+ assertEquals(0, controlBinaryDocValues.nextDoc());
+ assertEquals(controlBinaryDocValues.binaryValue(), binaryDocValues.binaryValue());
SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
SortedDocValues controlSortedDocValues = controlLeafReader.getSortedDocValues("sorted");
assertEquals(controlSortedDocValues.getValueCount(), sortedDocValues.getValueCount());
- assertEquals(controlSortedDocValues.get(0), sortedDocValues.get(0));
- assertEquals(controlSortedDocValues.getOrd(0), sortedDocValues.getOrd(0));
+ assertEquals(0, sortedDocValues.nextDoc());
+ assertEquals(0, controlSortedDocValues.nextDoc());
+ assertEquals(controlSortedDocValues.binaryValue(), sortedDocValues.binaryValue());
+ assertEquals(controlSortedDocValues.ordValue(), sortedDocValues.ordValue());
assertEquals(controlSortedDocValues.lookupOrd(0), sortedDocValues.lookupOrd(0));
SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
- sortedSetDocValues.setDocument(0);
+ assertEquals(0, sortedSetDocValues.nextDoc());
SortedSetDocValues controlSortedSetDocValues = controlLeafReader.getSortedSetDocValues("sorted_set");
- controlSortedSetDocValues.setDocument(0);
+ assertEquals(0, controlSortedSetDocValues.nextDoc());
assertEquals(controlSortedSetDocValues.getValueCount(), sortedSetDocValues.getValueCount());
for (long controlOrd = controlSortedSetDocValues.nextOrd(); controlOrd != SortedSetDocValues.NO_MORE_ORDS;
controlOrd = controlSortedSetDocValues.nextOrd()) {
@@ -563,7 +553,11 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
IndexReader controlIndexReader = DirectoryReader.open(dir);
LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader();
- assertEquals(controlLeafReader.getNormValues("text").get(0), leafReader.getNormValues("text").get(0));
+ NumericDocValues norms = controlLeafReader.getNormValues("text");
+ assertEquals(0, norms.nextDoc());
+ NumericDocValues norms2 = leafReader.getNormValues("text");
+ assertEquals(0, norms2.nextDoc());
+ assertEquals(norms.longValue(), norms2.longValue());
controlIndexReader.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/misc/src/java/org/apache/lucene/search/DiversifiedTopDocsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/search/DiversifiedTopDocsCollector.java b/lucene/misc/src/java/org/apache/lucene/search/DiversifiedTopDocsCollector.java
index f2016cc..6b75622 100644
--- a/lucene/misc/src/java/org/apache/lucene/search/DiversifiedTopDocsCollector.java
+++ b/lucene/misc/src/java/org/apache/lucene/search/DiversifiedTopDocsCollector.java
@@ -110,7 +110,7 @@ public abstract class DiversifiedTopDocsCollector extends
}
protected ScoreDocKey insert(ScoreDocKey addition, int docBase,
- NumericDocValues keys) {
+ NumericDocValues keys) throws IOException {
if ((globalQueue.size() >= numHits)
&& (globalQueue.lessThan(addition, globalQueue.top()))) {
// Queue is full and proposed addition is not a globally
@@ -122,7 +122,17 @@ public abstract class DiversifiedTopDocsCollector extends
// We delay fetching the key until we are certain the score is globally
// competitive. We need to adjust the ScoreDoc's global doc value to be
// a leaf reader value when looking up keys
- addition.key = keys.get(addition.doc - docBase);
+ int leafDocID = addition.doc - docBase;
+ long value;
+ if (keys.docID() < leafDocID) {
+ keys.advance(leafDocID);
+ }
+ if (keys.docID() == leafDocID) {
+ value = keys.longValue();
+ } else {
+ value = 0;
+ }
+ addition.key = value;
// For this to work the choice of key class needs to implement
// hashcode and equals.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
index 05a3b23..185b7ab 100644
--- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
+++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java
@@ -127,11 +127,32 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
protected NumericDocValues getKeys(final LeafReaderContext context) {
return new NumericDocValues() {
+
+ @Override
+ public int docID() {
+ return sdv.docID() - context.docBase;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return sdv.nextDoc() - context.docBase;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return sdv.advance(target + context.docBase);
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
@Override
- public long get(int docID) {
+ public long longValue() {
// Keys are always expressed as a long so we obtain the
// ordinal for our String-based artist name here
- return sdv.getOrd(context.docBase + docID);
+ return sdv.ordValue();
}
};
}
@@ -139,8 +160,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
// Alternative, faster implementation for converting String keys to longs
// but with the potential for hash collisions
- private static final class HashedDocValuesDiversifiedCollector extends
- DiversifiedTopDocsCollector {
+ private static final class HashedDocValuesDiversifiedCollector extends DiversifiedTopDocsCollector {
private final String field;
private BinaryDocValues vals;
@@ -155,8 +175,24 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
protected NumericDocValues getKeys(LeafReaderContext context) {
return new NumericDocValues() {
@Override
- public long get(int docID) {
- return vals == null ? -1 : vals.get(docID).hashCode();
+ public int docID() {
+ return vals.docID();
+ }
+ @Override
+ public int nextDoc() throws IOException {
+ return vals.nextDoc();
+ }
+ @Override
+ public int advance(int target) throws IOException {
+ return vals.advance(target);
+ }
+ @Override
+ public long cost() {
+ return vals.cost();
+ }
+ @Override
+ public long longValue() {
+ return vals == null ? -1 : vals.binaryValue().hashCode();
}
};
}
@@ -277,7 +313,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
private DiversifiedTopDocsCollector doDiversifiedSearch(int numResults,
int maxResultsPerArtist) throws IOException {
- // Alternate between implementations used for key lookups
+ // Alternate between implementations used for key lookups
if (random().nextBoolean()) {
// Faster key lookup but with potential for collisions on larger datasets
return doFuzzyDiversifiedSearch(numResults, maxResultsPerArtist);
@@ -427,13 +463,19 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
public SimScorer simScorer(SimWeight stats, LeafReaderContext context)
throws IOException {
final SimScorer sub = sim.simScorer(stats, context);
- final NumericDocValues values = DocValues.getNumeric(context.reader(),
- scoreValueField);
+ final NumericDocValues values = DocValues.getNumeric(context.reader(), scoreValueField);
return new SimScorer() {
@Override
- public float score(int doc, float freq) {
- return Float.intBitsToFloat((int) values.get(doc));
+ public float score(int doc, float freq) throws IOException {
+ if (doc != values.docID()) {
+ values.advance(doc);
+ }
+ if (doc == values.docID()) {
+ return Float.intBitsToFloat((int) values.longValue());
+ } else {
+ return 0f;
+ }
}
@Override
@@ -448,12 +490,10 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
}
@Override
- public Explanation explain(int doc, Explanation freq) {
- return Explanation.match(Float.intBitsToFloat((int) values.get(doc)),
- "indexDocValue(" + scoreValueField + ")");
+ public Explanation explain(int doc, Explanation freq) throws IOException {
+ return Explanation.match(score(doc, 0f), "indexDocValue(" + scoreValueField + ")");
}
};
}
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
index 1e7590d..9e73b4b 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function;
+import java.io.IOException;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Scorer;
@@ -38,22 +40,22 @@ import org.apache.lucene.util.mutable.MutableValueFloat;
// want the Query carrying around big objects
public abstract class FunctionValues {
- public byte byteVal(int doc) { throw new UnsupportedOperationException(); }
- public short shortVal(int doc) { throw new UnsupportedOperationException(); }
+ public byte byteVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
+ public short shortVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
- public float floatVal(int doc) { throw new UnsupportedOperationException(); }
- public int intVal(int doc) { throw new UnsupportedOperationException(); }
- public long longVal(int doc) { throw new UnsupportedOperationException(); }
- public double doubleVal(int doc) { throw new UnsupportedOperationException(); }
+ public float floatVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
+ public int intVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
+ public long longVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
+ public double doubleVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
// TODO: should we make a termVal, returns BytesRef?
- public String strVal(int doc) { throw new UnsupportedOperationException(); }
+ public String strVal(int doc) throws IOException { throw new UnsupportedOperationException(); }
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return intVal(doc) != 0;
}
/** returns the bytes representation of the string val - TODO: should this return the indexed raw bytes not? */
- public boolean bytesVal(int doc, BytesRefBuilder target) {
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
String s = strVal(doc);
if (s==null) {
target.clear();
@@ -64,13 +66,13 @@ public abstract class FunctionValues {
}
/** Native Java Object representation of the value */
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
// most FunctionValues are functions, so by default return a Float()
return floatVal(doc);
}
/** Returns true if there is a value for this document */
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return true;
}
@@ -79,13 +81,15 @@ public abstract class FunctionValues {
* @return the sort ordinal for the specified doc
* TODO: Maybe we can just use intVal for this...
*/
- public int ordVal(int doc) { throw new UnsupportedOperationException(); }
+ public int ordVal(int doc) throws IOException {
+ throw new UnsupportedOperationException();
+ }
/**
* @return the number of unique sort ordinals this instance has
*/
public int numOrd() { throw new UnsupportedOperationException(); }
- public abstract String toString(int doc);
+ public abstract String toString(int doc) throws IOException;
/**
* Abstraction of the logic required to fill the value of a specified doc into
@@ -100,7 +104,7 @@ public abstract class FunctionValues {
public abstract MutableValue getValue();
/** MutableValue will be reused across calls. Returns true if the value exists. */
- public abstract void fillValue(int doc);
+ public abstract void fillValue(int doc) throws IOException;
}
/** @lucene.experimental */
@@ -114,25 +118,25 @@ public abstract class FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = floatVal(doc);
}
};
}
//For Functions that can work with multiple values from the same document. This does not apply to all functions
- public void byteVal(int doc, byte [] vals) { throw new UnsupportedOperationException(); }
- public void shortVal(int doc, short [] vals) { throw new UnsupportedOperationException(); }
+ public void byteVal(int doc, byte [] vals) throws IOException { throw new UnsupportedOperationException(); }
+ public void shortVal(int doc, short [] vals) throws IOException { throw new UnsupportedOperationException(); }
- public void floatVal(int doc, float [] vals) { throw new UnsupportedOperationException(); }
- public void intVal(int doc, int [] vals) { throw new UnsupportedOperationException(); }
- public void longVal(int doc, long [] vals) { throw new UnsupportedOperationException(); }
- public void doubleVal(int doc, double [] vals) { throw new UnsupportedOperationException(); }
+ public void floatVal(int doc, float [] vals) throws IOException { throw new UnsupportedOperationException(); }
+ public void intVal(int doc, int [] vals) throws IOException { throw new UnsupportedOperationException(); }
+ public void longVal(int doc, long [] vals) throws IOException { throw new UnsupportedOperationException(); }
+ public void doubleVal(int doc, double [] vals) throws IOException { throw new UnsupportedOperationException(); }
// TODO: should we make a termVal, fills BytesRef[]?
- public void strVal(int doc, String [] vals) { throw new UnsupportedOperationException(); }
+ public void strVal(int doc, String [] vals) throws IOException { throw new UnsupportedOperationException(); }
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
return Explanation.match(floatVal(doc), toString(doc));
}
@@ -157,7 +161,7 @@ public abstract class FunctionValues {
// because it needs different behavior depending on the type of fields. There is also
// a setup cost - parsing and normalizing params, and doing a binary search on the StringIndex.
// TODO: change "reader" to LeafReaderContext
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
float lower;
float upper;
@@ -178,7 +182,7 @@ public abstract class FunctionValues {
if (includeLower && includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
float docVal = floatVal(doc);
return docVal >= l && docVal <= u;
@@ -188,7 +192,7 @@ public abstract class FunctionValues {
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
float docVal = floatVal(doc);
return docVal >= l && docVal < u;
@@ -198,7 +202,7 @@ public abstract class FunctionValues {
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
float docVal = floatVal(doc);
return docVal > l && docVal <= u;
@@ -208,7 +212,7 @@ public abstract class FunctionValues {
else {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
float docVal = floatVal(doc);
return docVal > l && docVal < u;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
index 49d4b77..4064fc9 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
@@ -38,7 +38,9 @@ public abstract class ValueSource {
/**
* Gets the values for this reader and the context that was previously
- * passed to createWeight()
+ * passed to createWeight(). The values must be consumed in a forward
+ * docID manner, and you must call this method again to iterate through
+ * the values again.
*/
public abstract FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException;
@@ -144,12 +146,12 @@ public abstract class ValueSource {
}
@Override
- public int compareBottom(int doc) {
+ public int compareBottom(int doc) throws IOException {
return Double.compare(bottom, docVals.doubleVal(doc));
}
@Override
- public void copy(int slot, int doc) {
+ public void copy(int slot, int doc) throws IOException {
values[slot] = docVals.doubleVal(doc);
}
@@ -174,7 +176,7 @@ public abstract class ValueSource {
}
@Override
- public int compareTop(int doc) {
+ public int compareTop(int doc) throws IOException {
final double docValue = docVals.doubleVal(doc);
return Double.compare(topValue, docValue);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
index 035327b..c9303cd 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
@@ -62,7 +62,7 @@ public abstract class ValueSourceScorer extends Scorer {
}
/** Override to decide if this document matches. It's called by {@link TwoPhaseIterator#matches()}. */
- public abstract boolean matches(int doc);
+ public abstract boolean matches(int doc) throws IOException;
@Override
public DocIdSetIterator iterator() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java
index f3066e3..eacea1d 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.mutable.MutableValue;
@@ -33,50 +35,50 @@ public abstract class BoolDocValues extends FunctionValues {
}
@Override
- public abstract boolean boolVal(int doc);
+ public abstract boolean boolVal(int doc) throws IOException;
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return boolVal(doc) ? (byte)1 : (byte)0;
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return boolVal(doc) ? (short)1 : (short)0;
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return boolVal(doc) ? (float)1 : (float)0;
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return boolVal(doc) ? 1 : 0;
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return boolVal(doc) ? (long)1 : (long)0;
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return boolVal(doc) ? (double)1 : (double)0;
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Boolean.toString(boolVal(doc));
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? boolVal(doc) : null;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -91,7 +93,7 @@ public abstract class BoolDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = boolVal(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
index 194969d..e855e27 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
@@ -39,26 +39,45 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
protected final ValueSource vs;
protected final MutableValueStr val = new MutableValueStr();
protected final CharsRefBuilder spareChars = new CharsRefBuilder();
+ private final String field;
+ private int lastDocID;
public DocTermsIndexDocValues(ValueSource vs, LeafReaderContext context, String field) throws IOException {
- this(vs, open(context, field));
+ this(field, vs, open(context, field));
}
- protected DocTermsIndexDocValues(ValueSource vs, SortedDocValues termsIndex) {
+ protected DocTermsIndexDocValues(String field, ValueSource vs, SortedDocValues termsIndex) {
+ this.field = field;
this.vs = vs;
this.termsIndex = termsIndex;
}
+ protected int getOrdForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = termsIndex.docID();
+ if (doc > curDocID) {
+ curDocID = termsIndex.advance(doc);
+ }
+ if (doc == curDocID) {
+ return termsIndex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
protected abstract String toTerm(String readableValue);
@Override
- public boolean exists(int doc) {
- return ordVal(doc) >= 0;
+ public boolean exists(int doc) throws IOException {
+ return getOrdForDoc(doc) >= 0;
}
@Override
- public int ordVal(int doc) {
- return termsIndex.getOrd(doc);
+ public int ordVal(int doc) throws IOException {
+ return getOrdForDoc(doc);
}
@Override
@@ -67,32 +86,36 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder target) {
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
target.clear();
- target.copyBytes(termsIndex.get(doc));
- return target.length() > 0;
+ if (getOrdForDoc(doc) == -1) {
+ return false;
+ } else {
+ target.copyBytes(termsIndex.binaryValue());
+ return true;
+ }
}
@Override
- public String strVal(int doc) {
- final BytesRef term = termsIndex.get(doc);
- if (term.length == 0) {
+ public String strVal(int doc) throws IOException {
+ if (getOrdForDoc(doc) == -1) {
return null;
}
+ final BytesRef term = termsIndex.binaryValue();
spareChars.copyUTF8Bytes(term);
return spareChars.toString();
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return exists(doc);
}
@Override
- public abstract Object objectVal(int doc); // force subclasses to override
+ public abstract Object objectVal(int doc) throws IOException; // force subclasses to override
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
// TODO: are lowerVal and upperVal in indexed form or not?
lowerVal = lowerVal == null ? null : toTerm(lowerVal);
upperVal = upperVal == null ? null : toTerm(upperVal);
@@ -121,16 +144,29 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
final int uu = upper;
return new ValueSourceScorer(readerContext, this) {
+ final SortedDocValues values = readerContext.reader().getSortedDocValues(field);
+ private int lastDocID;
+
@Override
- public boolean matches(int doc) {
- int ord = termsIndex.getOrd(doc);
- return ord >= ll && ord <= uu;
+ public boolean matches(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ if (doc > values.docID()) {
+ values.advance(doc);
+ }
+ if (doc == values.docID()) {
+ int ord = values.ordValue();
+ return ord >= ll && ord <= uu;
+ } else {
+ return false;
+ }
}
};
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -145,8 +181,8 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
- int ord = termsIndex.getOrd(doc);
+ public void fillValue(int doc) throws IOException {
+ int ord = getOrdForDoc(doc);
mval.value.clear();
mval.exists = ord >= 0;
if (mval.exists) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
index 91d1e6f..646db9c 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,50 +37,50 @@ public abstract class DoubleDocValues extends FunctionValues {
}
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return (byte)doubleVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return (short)doubleVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float)doubleVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return (int)doubleVal(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return (long)doubleVal(doc);
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return doubleVal(doc) != 0;
}
@Override
- public abstract double doubleVal(int doc);
+ public abstract double doubleVal(int doc) throws IOException;
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Double.toString(doubleVal(doc));
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? doubleVal(doc) : null;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -105,7 +107,7 @@ public abstract class DoubleDocValues extends FunctionValues {
if (includeLower && includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
double docVal = doubleVal(doc);
return docVal >= l && docVal <= u;
@@ -115,7 +117,7 @@ public abstract class DoubleDocValues extends FunctionValues {
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
double docVal = doubleVal(doc);
return docVal >= l && docVal < u;
@@ -125,7 +127,7 @@ public abstract class DoubleDocValues extends FunctionValues {
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
double docVal = doubleVal(doc);
return docVal > l && docVal <= u;
@@ -135,7 +137,7 @@ public abstract class DoubleDocValues extends FunctionValues {
else {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
double docVal = doubleVal(doc);
return docVal > l && docVal < u;
@@ -155,7 +157,7 @@ public abstract class DoubleDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = doubleVal(doc);
mval.exists = exists(doc);
}
[03/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
index 2be8d1e..0c88b80 100644
--- a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
+++ b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
@@ -16,6 +16,16 @@
*/
package org.apache.solr.request;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
@@ -36,16 +46,6 @@ import org.apache.solr.search.Filter;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.BoundedTreeSet;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-
class PerSegmentSingleValuedFaceting {
@@ -293,7 +293,15 @@ class PerSegmentSingleValuedFaceting {
// specialized version when collecting counts for all terms
int doc;
while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
- int t = 1+si.getOrd(doc);
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ int t;
+ if (doc == si.docID()) {
+ t = 1+si.ordValue();
+ } else {
+ t = 0;
+ }
hasAnyCount = hasAnyCount || t > 0; //counts[0] == missing counts
counts[t]++;
}
@@ -301,7 +309,15 @@ class PerSegmentSingleValuedFaceting {
// version that adjusts term numbers because we aren't collecting the full range
int doc;
while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
- int term = si.getOrd(doc);
+ if (doc > si.docID()) {
+ si.advance(doc);
+ }
+ int term;
+ if (doc == si.docID()) {
+ term = si.ordValue();
+ } else {
+ term = -1;
+ }
int arrIdx = term-startTermIndex;
if (arrIdx>=0 && arrIdx<nTerms){
counts[arrIdx]++;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
index b3752a8..709abc9 100644
--- a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
@@ -20,9 +20,9 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
-import java.util.ArrayList;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
@@ -857,6 +857,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
protected String field;
protected int currentValue;
protected IntComp comp;
+ private int lastDocID;
public IntValue copy() {
return new IntValue(field, comp);
@@ -872,8 +873,20 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.vals = DocValues.getNumeric(context.reader(), field);
}
- public void setCurrentValue(int docId) {
- currentValue = (int)vals.get(docId);
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = (int) vals.longValue();
+ } else {
+ currentValue = 0;
+ }
}
public int compareTo(SortValue o) {
@@ -935,6 +948,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
protected String field;
protected long currentValue;
protected LongComp comp;
+ private int lastDocID;
public LongValue(String field, LongComp comp) {
this.field = field;
@@ -950,8 +964,20 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.vals = DocValues.getNumeric(context.reader(), field);
}
- public void setCurrentValue(int docId) {
- currentValue = vals.get(docId);
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = vals.longValue();
+ } else {
+ currentValue = 0;
+ }
}
public void setCurrentValue(SortValue sv) {
@@ -1014,6 +1040,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
protected String field;
protected float currentValue;
protected FloatComp comp;
+ private int lastDocID;
public FloatValue(String field, FloatComp comp) {
this.field = field;
@@ -1029,8 +1056,20 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.vals = DocValues.getNumeric(context.reader(), field);
}
- public void setCurrentValue(int docId) {
- currentValue = Float.intBitsToFloat((int)vals.get(docId));
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = Float.intBitsToFloat((int)vals.longValue());
+ } else {
+ currentValue = 0f;
+ }
}
public void setCurrentValue(SortValue sv) {
@@ -1091,6 +1130,8 @@ public class SortingResponseWriter implements QueryResponseWriter {
protected String field;
protected double currentValue;
protected DoubleComp comp;
+ private int lastDocID;
+ private LeafReader reader;
public DoubleValue(String field, DoubleComp comp) {
this.field = field;
@@ -1103,11 +1144,25 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = DocValues.getNumeric(context.reader(), field);
+ this.reader = context.reader();
+ this.vals = DocValues.getNumeric(this.reader, this.field);
}
- public void setCurrentValue(int docId) {
- currentValue = Double.longBitsToDouble(vals.get(docId));
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ // TODO: can we enforce caller to go in order instead?
+ this.vals = DocValues.getNumeric(this.reader, this.field);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = Double.longBitsToDouble(vals.longValue());
+ } else {
+ currentValue = 0f;
+ }
}
public void setCurrentValue(SortValue sv) {
@@ -1178,7 +1233,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
public StringValue(SortedDocValues vals, String field, IntComp comp) {
this.vals = vals;
- if(vals instanceof MultiDocValues.MultiSortedDocValues) {
+ if(vals instanceof MultiDocValues.MultiSortedDocValues) {
this.segmentVals = ((MultiDocValues.MultiSortedDocValues) vals).values;
this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) vals).mapping;
}
@@ -1191,17 +1246,19 @@ public class SortingResponseWriter implements QueryResponseWriter {
return new StringValue(vals, field, comp);
}
- public void setCurrentValue(int docId) {
- int ord = currentVals.getOrd(docId);
-
- if(ord < 0) {
- currentOrd = -1;
- } else {
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId > currentVals.docID()) {
+ currentVals.advance(docId);
+ }
+ if (docId == currentVals.docID()) {
+ int ord = currentVals.ordValue();
if(globalOrds != null) {
currentOrd = (int)globalOrds.get(ord);
} else {
currentOrd = ord;
}
+ } else {
+ currentOrd = -1;
}
}
@@ -1247,7 +1304,12 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- int val = (int)vals.get(docId);
+ int val;
+ if (vals.advance(docId) == docId) {
+ val = (int) vals.longValue();
+ } else {
+ val = 0;
+ }
if(fieldIndex>0) {
out.write(',');
}
@@ -1273,14 +1335,15 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
- vals.setDocument(docId);
- List<Long> ords = new ArrayList();
- long o = -1;
- while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- ords.add(o);
- }
-
- if(ords.size()== 0) {
+ List<Long> ords;
+ if (vals.advance(docId) == docId) {
+ ords = new ArrayList();
+ long o = -1;
+ while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ ords.add(o);
+ }
+ assert ords.size() > 0;
+ } else {
return false;
}
@@ -1326,7 +1389,12 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- long val = vals.get(docId);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ val = 0;
+ }
if(fieldIndex > 0) {
out.write(',');
}
@@ -1348,7 +1416,12 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- long val = vals.get(docId);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ val = 0;
+ }
if (fieldIndex > 0) {
out.write(',');
@@ -1376,10 +1449,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
- int ord = vals.getOrd(docId);
- if(ord == -1) {
+ if (vals.advance(docId) != docId) {
return false;
}
+ int ord = vals.ordValue();
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
@@ -1407,7 +1480,12 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- int val = (int)vals.get(docId);
+ int val;
+ if (vals.advance(docId) == docId) {
+ val = (int)vals.longValue();
+ } else {
+ val = 0;
+ }
if(fieldIndex > 0) {
out.write(',');
}
@@ -1429,10 +1507,15 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ val = 0;
+ }
if(fieldIndex > 0) {
out.write(',');
}
- long val = vals.get(docId);
out.write('"');
out.write(this.field);
out.write('"');
@@ -1454,10 +1537,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
- int ord = vals.getOrd(docId);
- if(ord == -1) {
+ if (vals.advance(docId) != docId) {
return false;
}
+ int ord = vals.ordValue();
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
index 9edf826..c37dd80 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
@@ -65,7 +65,6 @@ public class ValueSourceAugmenter extends DocTransformer
try {
searcher = context.getSearcher();
readerContexts = searcher.getIndexReader().leaves();
- docValuesArr = new FunctionValues[readerContexts.size()];
fcontext = ValueSource.newContext(searcher);
this.valueSource.createWeight(fcontext, searcher);
} catch (IOException e) {
@@ -76,7 +75,6 @@ public class ValueSourceAugmenter extends DocTransformer
Map fcontext;
SolrIndexSearcher searcher;
List<LeafReaderContext> readerContexts;
- FunctionValues docValuesArr[];
@Override
public void transform(SolrDocument doc, int docid, float score) {
@@ -87,11 +85,7 @@ public class ValueSourceAugmenter extends DocTransformer
// TODO: calculate this stuff just once across diff functions
int idx = ReaderUtil.subIndex(docid, readerContexts);
LeafReaderContext rcontext = readerContexts.get(idx);
- FunctionValues values = docValuesArr[idx];
- if (values == null) {
- docValuesArr[idx] = values = valueSource.getValues(fcontext, rcontext);
- }
-
+ FunctionValues values = valueSource.getValues(fcontext, rcontext);
int localId = docid - rcontext.docBase;
setValue(doc,values.objectVal(localId));
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/BoolField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/BoolField.java b/solr/core/src/java/org/apache/solr/schema/BoolField.java
index a9acfc5..210ea0b 100644
--- a/solr/core/src/java/org/apache/solr/schema/BoolField.java
+++ b/solr/core/src/java/org/apache/solr/schema/BoolField.java
@@ -245,14 +245,26 @@ class BoolFieldSource extends ValueSource {
final int trueOrd = tord;
return new BoolDocValues(this) {
+
+ private int getOrdForDoc(int doc) throws IOException {
+ if (doc > sindex.docID()) {
+ sindex.advance(doc);
+ }
+ if (doc == sindex.docID()) {
+ return sindex.ordValue();
+ } else {
+ return -1;
+ }
+ }
@Override
- public boolean boolVal(int doc) {
- return sindex.getOrd(doc) == trueOrd;
+
+ public boolean boolVal(int doc) throws IOException {
+ return getOrdForDoc(doc) == trueOrd;
}
@Override
- public boolean exists(int doc) {
- return sindex.getOrd(doc) != -1;
+ public boolean exists(int doc) throws IOException {
+ return getOrdForDoc(doc) != -1;
}
@Override
@@ -266,8 +278,8 @@ class BoolFieldSource extends ValueSource {
}
@Override
- public void fillValue(int doc) {
- int ord = sindex.getOrd(doc);
+ public void fillValue(int doc) throws IOException {
+ int ord = getOrdForDoc(doc);
mval.value = (ord == trueOrd);
mval.exists = (ord != -1);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
index 26ea167..f2dc351 100644
--- a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
+++ b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
@@ -398,35 +398,35 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
final double divisor = Math.pow(10D, targetCurrency.getDefaultFractionDigits());
return new FunctionValues() {
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return amounts.exists(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return (long) doubleVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return (int) doubleVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return CurrencyValue.convertAmount(rate, sourceCurrencyCode, amounts.longVal(doc), targetCurrency.getCurrencyCode()) / divisor;
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return CurrencyValue.convertAmount(rate, sourceCurrencyCode, amounts.longVal(doc), targetCurrency.getCurrencyCode()) / ((float)divisor);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Double.toString(doubleVal(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + strVal(doc) + ')';
}
};
@@ -514,7 +514,7 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
private int targetCurrencyOrd = -1;
private boolean initializedCache;
- private String getDocCurrencyCode(int doc, int currencyOrd) {
+ private String getDocCurrencyCode(int doc, int currencyOrd) throws IOException {
if (currencyOrd < MAX_CURRENCIES_TO_CACHE) {
String currency = currencyOrdToCurrencyCache[currencyOrd];
@@ -537,7 +537,7 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
}
}
/** throws a (Server Error) SolrException if the code is not valid */
- private Currency getDocCurrency(int doc, int currencyOrd) {
+ private Currency getDocCurrency(int doc, int currencyOrd) throws IOException {
String code = getDocCurrencyCode(doc, currencyOrd);
Currency c = getCurrency(code);
if (null == c) {
@@ -549,12 +549,12 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return amounts.exists(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
long amount = amounts.longVal(doc);
// bail fast using whatever amounts defaults to if no value
// (if we don't do this early, currencyOrd may be < 0,
@@ -607,27 +607,27 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return (int) longVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return (double) longVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float) longVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Long.toString(longVal(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + amounts.toString(doc) + ',' + currencies.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/LatLonType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java
index b6d1b52..53935b8 100644
--- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java
+++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java
@@ -378,7 +378,7 @@ class SpatialDistanceQuery extends ExtendedQueryBase implements PostFilter {
}
- boolean match() {
+ boolean match() throws IOException {
// longitude should generally be more restrictive than latitude
// (e.g. in the US, it immediately separates the coasts, and in world search separates
// US from Europe from Asia, etc.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
index ca2771c..f072867 100644
--- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
+++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
@@ -156,7 +156,7 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
int docId = -1;
Shape shape = null;
- private void setShapeFromDoc(int doc) {
+ private void setShapeFromDoc(int doc) throws IOException {
if (docId == doc) {
return;
}
@@ -180,24 +180,24 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
// Use the cache for exists & objectVal;
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
setShapeFromDoc(doc);
return shape != null;
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
setShapeFromDoc(doc);
return shape;
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
return targetFuncValues.explain(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return targetFuncValues.toString(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java b/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
index f2a5075..5326a76 100644
--- a/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
+++ b/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
@@ -45,8 +45,8 @@ public class StrFieldSource extends FieldCacheSource {
}
@Override
- public int ordVal(int doc) {
- return termsIndex.getOrd(doc);
+ public int ordVal(int doc) throws IOException {
+ return getOrdForDoc(doc);
}
@Override
@@ -55,12 +55,12 @@ public class StrFieldSource extends FieldCacheSource {
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return strVal(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + '=' + strVal(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java b/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java
index cb409a1..3891602 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java
@@ -19,13 +19,13 @@ package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.legacy.LegacyNumericUtils;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortedSetSelector;
@@ -75,20 +75,33 @@ public class TrieDoubleField extends TrieField implements DoubleValueFieldType {
SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
return new DoubleDocValues(thisAsSortedSetFieldSource) {
+ private int lastDocID;
+
+ private boolean setDoc(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new IllegalArgumentException("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > view.docID()) {
+ return docID == view.advance(docID);
+ } else {
+ return docID == view.docID();
+ }
+ }
+
@Override
- public double doubleVal(int doc) {
- BytesRef bytes = view.get(doc);
- if (0 == bytes.length) {
- // the only way this should be possible is for non existent value
- assert !exists(doc) : "zero bytes for doc, but exists is true";
+ public double doubleVal(int doc) throws IOException {
+ if (setDoc(doc)) {
+ BytesRef bytes = view.binaryValue();
+ assert bytes.length > 0;
+ return NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(bytes));
+ } else {
return 0D;
}
- return NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(bytes));
}
@Override
- public boolean exists(int doc) {
- return -1 != view.getOrd(doc);
+ public boolean exists(int doc) throws IOException {
+ return setDoc(doc);
}
@Override
@@ -102,13 +115,14 @@ public class TrieDoubleField extends TrieField implements DoubleValueFieldType {
}
@Override
- public void fillValue(int doc) {
- // micro optimized (eliminate at least one redundant ord check)
- //mval.exists = exists(doc);
- //mval.value = mval.exists ? doubleVal(doc) : 0.0D;
- BytesRef bytes = view.get(doc);
- mval.exists = (0 == bytes.length);
- mval.value = mval.exists ? NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(bytes)) : 0D;
+ public void fillValue(int doc) throws IOException {
+ if (setDoc(doc)) {
+ mval.exists = true;
+ mval.value = NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(view.binaryValue()));
+ } else {
+ mval.exists = false;
+ mval.value = 0D;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java
index bcc4d4f..13b9141 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java
@@ -19,13 +19,13 @@ package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.legacy.LegacyNumericUtils;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortedSetSelector;
@@ -75,20 +75,33 @@ public class TrieFloatField extends TrieField implements FloatValueFieldType {
SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
return new FloatDocValues(thisAsSortedSetFieldSource) {
+ private int lastDocID;
+
+ private boolean setDoc(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new IllegalArgumentException("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > view.docID()) {
+ return docID == view.advance(docID);
+ } else {
+ return docID == view.docID();
+ }
+ }
+
@Override
- public float floatVal(int doc) {
- BytesRef bytes = view.get(doc);
- if (0 == bytes.length) {
- // the only way this should be possible is for non existent value
- assert !exists(doc) : "zero bytes for doc, but exists is true";
+ public float floatVal(int doc) throws IOException {
+ if (setDoc(doc)) {
+ BytesRef bytes = view.binaryValue();
+ assert bytes.length > 0;
+ return NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(bytes));
+ } else {
return 0F;
}
- return NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(bytes));
}
@Override
- public boolean exists(int doc) {
- return -1 != view.getOrd(doc);
+ public boolean exists(int doc) throws IOException {
+ return setDoc(doc);
}
@Override
@@ -102,14 +115,14 @@ public class TrieFloatField extends TrieField implements FloatValueFieldType {
}
@Override
- public void fillValue(int doc) {
- // micro optimized (eliminate at least one redundant ord check)
- //mval.exists = exists(doc);
- //mval.value = mval.exists ? floatVal(doc) : 0.0F;
- //
- BytesRef bytes = view.get(doc);
- mval.exists = (0 == bytes.length);
- mval.value = mval.exists ? NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(bytes)) : 0F;
+ public void fillValue(int doc) throws IOException {
+ if (setDoc(doc)) {
+ mval.exists = true;
+ mval.value = NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(view.binaryValue()));
+ } else {
+ mval.exists = false;
+ mval.value = 0F;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/TrieIntField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java
index 6c8622b..dedb667 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java
@@ -19,13 +19,13 @@ package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.legacy.LegacyNumericUtils;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortedSetSelector;
@@ -73,20 +73,33 @@ public class TrieIntField extends TrieField implements IntValueFieldType {
SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
return new IntDocValues(thisAsSortedSetFieldSource) {
+ private int lastDocID;
+
+ private boolean setDoc(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new IllegalArgumentException("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > view.docID()) {
+ return docID == view.advance(docID);
+ } else {
+ return docID == view.docID();
+ }
+ }
+
@Override
- public int intVal(int doc) {
- BytesRef bytes = view.get(doc);
- if (0 == bytes.length) {
- // the only way this should be possible is for non existent value
- assert !exists(doc) : "zero bytes for doc, but exists is true";
+ public int intVal(int doc) throws IOException {
+ if (setDoc(doc)) {
+ BytesRef bytes = view.binaryValue();
+ assert bytes.length > 0;
+ return LegacyNumericUtils.prefixCodedToInt(bytes);
+ } else {
return 0;
}
- return LegacyNumericUtils.prefixCodedToInt(bytes);
}
@Override
- public boolean exists(int doc) {
- return -1 != view.getOrd(doc);
+ public boolean exists(int doc) throws IOException {
+ return setDoc(doc);
}
@Override
@@ -100,14 +113,14 @@ public class TrieIntField extends TrieField implements IntValueFieldType {
}
@Override
- public void fillValue(int doc) {
- // micro optimized (eliminate at least one redundant ord check)
- //mval.exists = exists(doc);
- //mval.value = mval.exists ? intVal(doc) : 0;
- //
- BytesRef bytes = view.get(doc);
- mval.exists = (0 == bytes.length);
- mval.value = mval.exists ? LegacyNumericUtils.prefixCodedToInt(bytes) : 0;
+ public void fillValue(int doc) throws IOException {
+ if (setDoc(doc)) {
+ mval.exists = true;
+ mval.value = LegacyNumericUtils.prefixCodedToInt(view.binaryValue());
+ } else {
+ mval.exists = false;
+ mval.value = 0;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/schema/TrieLongField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java
index 371d5bb..229900c 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java
@@ -19,13 +19,13 @@ package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.legacy.LegacyNumericUtils;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.LongDocValues;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortedSetSelector;
@@ -73,20 +73,33 @@ public class TrieLongField extends TrieField implements LongValueFieldType {
SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
return new LongDocValues(thisAsSortedSetFieldSource) {
+ private int lastDocID;
+
+ private boolean setDoc(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new IllegalArgumentException("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > view.docID()) {
+ return docID == view.advance(docID);
+ } else {
+ return docID == view.docID();
+ }
+ }
+
@Override
- public long longVal(int doc) {
- BytesRef bytes = view.get(doc);
- if (0 == bytes.length) {
- // the only way this should be possible is for non existent value
- assert !exists(doc) : "zero bytes for doc, but exists is true";
+ public long longVal(int doc) throws IOException {
+ if (setDoc(doc)) {
+ BytesRef bytes = view.binaryValue();
+ assert bytes.length > 0;
+ return LegacyNumericUtils.prefixCodedToLong(bytes);
+ } else {
return 0L;
}
- return LegacyNumericUtils.prefixCodedToLong(bytes);
}
@Override
- public boolean exists(int doc) {
- return -1 != view.getOrd(doc);
+ public boolean exists(int doc) throws IOException {
+ return setDoc(doc);
}
@Override
@@ -100,14 +113,14 @@ public class TrieLongField extends TrieField implements LongValueFieldType {
}
@Override
- public void fillValue(int doc) {
- // micro optimized (eliminate at least one redundant ord check)
- //mval.exists = exists(doc);
- //mval.value = mval.exists ? longVal(doc) : 0;
- //
- BytesRef bytes = view.get(doc);
- mval.exists = (0 == bytes.length);
- mval.value = mval.exists ? LegacyNumericUtils.prefixCodedToLong(bytes) : 0L;
+ public void fillValue(int doc) throws IOException {
+ if (setDoc(doc)) {
+ mval.exists = true;
+ mval.value = LegacyNumericUtils.prefixCodedToLong(view.binaryValue());
+ } else {
+ mval.exists = false;
+ mval.value = 0L;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 56cde8f..4858d1c 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -26,8 +26,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.EmptyDocValuesProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterLeafReader;
@@ -459,6 +461,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private static class OrdScoreCollector extends DelegatingCollector {
private LeafReaderContext[] contexts;
+ private final DocValuesProducer collapseValuesProducer;
private FixedBitSet collapsedSet;
private SortedDocValues collapseValues;
private MultiDocValues.OrdinalMap ordinalMap;
@@ -479,13 +482,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public OrdScoreCollector(int maxDoc,
int segments,
- SortedDocValues collapseValues,
+ DocValuesProducer collapseValuesProducer,
int nullPolicy,
- IntIntHashMap boostDocsMap) {
+ IntIntHashMap boostDocsMap) throws IOException {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
this.collapsedSet = new FixedBitSet(maxDoc);
- this.collapseValues = collapseValues;
+ this.collapseValuesProducer = collapseValuesProducer;
+ this.collapseValues = collapseValuesProducer.getSorted(null);
+
int valueCount = collapseValues.getValueCount();
if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
@@ -537,13 +542,24 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int ord = -1;
if(this.ordinalMap != null) {
//Handle ordinalMapping case
- ord = segmentValues.getOrd(contextDoc);
- if(ord > -1) {
- ord = (int)segmentOrdinalMap.get(ord);
+ if (contextDoc > segmentValues.docID()) {
+ segmentValues.advance(contextDoc);
+ }
+ if (contextDoc == segmentValues.docID()) {
+ ord = (int)segmentOrdinalMap.get(segmentValues.ordValue());
+ } else {
+ ord = -1;
}
} else {
//Handle top Level FieldCache or Single Segment Case
- ord = segmentValues.getOrd(globalDoc);
+ if (globalDoc > segmentValues.docID()) {
+ segmentValues.advance(globalDoc);
+ }
+ if (globalDoc == segmentValues.docID()) {
+ ord = segmentValues.ordValue();
+ } else {
+ ord = -1;
+ }
}
// Check to see if we have documents boosted by the QueryElevationComponent
@@ -607,6 +623,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int currentContext = 0;
int currentDocBase = 0;
+ collapseValues = collapseValuesProducer.getSorted(null);
+
+ if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
+ this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
+ this.ordinalMap = multiSortedDocValues.mapping;
+ }
+
if(ordinalMap != null) {
this.segmentValues = this.multiSortedDocValues.values[currentContext];
this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
@@ -622,7 +645,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int docId = -1;
int index = -1;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-
while(docId >= nextDocBase) {
currentContext++;
currentDocBase = contexts[currentContext].docBase;
@@ -640,13 +662,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int ord = -1;
if(this.ordinalMap != null) {
//Handle ordinalMapping case
- ord = segmentValues.getOrd(contextDoc);
- if(ord > -1) {
- ord = (int)segmentOrdinalMap.get(ord);
+ if (contextDoc > segmentValues.docID()) {
+ segmentValues.advance(contextDoc);
+ }
+ if (contextDoc == segmentValues.docID()) {
+ ord = (int)segmentOrdinalMap.get(segmentValues.ordValue());
}
} else {
//Handle top Level FieldCache or Single Segment Case
- ord = segmentValues.getOrd(docId);
+ if (docId > segmentValues.docID()) {
+ segmentValues.advance(docId);
+ }
+ if (docId == segmentValues.docID()) {
+ ord = segmentValues.ordValue();
+ }
}
if(ord > -1) {
@@ -740,7 +769,17 @@ public class CollapsingQParserPlugin extends QParserPlugin {
@Override
public void collect(int contextDoc) throws IOException {
- int collapseValue = (int)this.collapseValues.get(contextDoc);
+ int collapseDocID = collapseValues.docID();
+ if (collapseDocID < contextDoc) {
+ collapseDocID = collapseValues.advance(contextDoc);
+ }
+
+ int collapseValue;
+ if (collapseDocID == contextDoc) {
+ collapseValue = (int) collapseValues.longValue();
+ } else {
+ collapseValue = 0;
+ }
int globalDoc = docBase+contextDoc;
// Check to see of we have documents boosted by the QueryElevationComponent
@@ -832,8 +871,17 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
int contextDoc = globalDoc-currentDocBase;
+ int valuesDocID = collapseValues.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = collapseValues.advance(contextDoc);
+ }
+ int collapseValue;
+ if (valuesDocID == contextDoc) {
+ collapseValue = (int) collapseValues.longValue();
+ } else {
+ collapseValue = 0;
+ }
- int collapseValue = (int)collapseValues.get(contextDoc);
if(collapseValue != nullValue) {
long scoreDoc = cmap.get(collapseValue);
dummy.score = Float.intBitsToFloat((int)(scoreDoc>>32));
@@ -860,6 +908,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
*/
private static class OrdFieldValueCollector extends DelegatingCollector {
private LeafReaderContext[] contexts;
+ private DocValuesProducer collapseValuesProducer;
private SortedDocValues collapseValues;
protected MultiDocValues.OrdinalMap ordinalMap;
protected SortedDocValues segmentValues;
@@ -874,7 +923,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public OrdFieldValueCollector(int maxDoc,
int segments,
- SortedDocValues collapseValues,
+ DocValuesProducer collapseValuesProducer,
int nullPolicy,
GroupHeadSelector groupHeadSelector,
SortSpec sortSpec,
@@ -887,7 +936,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
- this.collapseValues = collapseValues;
+ this.collapseValuesProducer = collapseValuesProducer;
+ this.collapseValues = collapseValuesProducer.getSorted(null);
if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
this.ordinalMap = multiSortedDocValues.mapping;
@@ -935,12 +985,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int globalDoc = contextDoc+this.docBase;
int ord = -1;
if(this.ordinalMap != null) {
- ord = segmentValues.getOrd(contextDoc);
- if(ord > -1) {
- ord = (int)segmentOrdinalMap.get(ord);
+ if (contextDoc > segmentValues.docID()) {
+ segmentValues.advance(contextDoc);
+ }
+ if (contextDoc == segmentValues.docID()) {
+ ord = (int)segmentOrdinalMap.get(segmentValues.ordValue());
}
} else {
- ord = segmentValues.getOrd(globalDoc);
+ if (globalDoc > segmentValues.docID()) {
+ segmentValues.advance(globalDoc);
+ }
+ if (globalDoc == segmentValues.docID()) {
+ ord = segmentValues.ordValue();
+ }
}
collapseStrategy.collapse(ord, contextDoc, globalDoc);
}
@@ -953,6 +1010,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int currentContext = 0;
int currentDocBase = 0;
+ this.collapseValues = collapseValuesProducer.getSorted(null);
+ if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
+ this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
+ this.ordinalMap = multiSortedDocValues.mapping;
+ }
if(ordinalMap != null) {
this.segmentValues = this.multiSortedDocValues.values[currentContext];
this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
@@ -974,6 +1036,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+
while(globalDoc >= nextDocBase) {
currentContext++;
currentDocBase = contexts[currentContext].docBase;
@@ -992,13 +1055,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int ord = -1;
if(this.ordinalMap != null) {
//Handle ordinalMapping case
- ord = segmentValues.getOrd(contextDoc);
- if(ord > -1) {
- ord = (int)segmentOrdinalMap.get(ord);
+ if (contextDoc > segmentValues.docID()) {
+ segmentValues.advance(contextDoc);
+ }
+ if (contextDoc == segmentValues.docID()) {
+ ord = (int) segmentOrdinalMap.get(segmentValues.ordValue());
}
} else {
//Handle top Level FieldCache or Single Segment Case
- ord = segmentValues.getOrd(globalDoc);
+ if (globalDoc > segmentValues.docID()) {
+ segmentValues.advance(globalDoc);
+ }
+ if (globalDoc == segmentValues.docID()) {
+ ord = segmentValues.ordValue();
+ }
}
if(ord > -1) {
@@ -1089,8 +1159,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
public void collect(int contextDoc) throws IOException {
+ int collapseDocID = collapseValues.docID();
+ if (collapseDocID < contextDoc) {
+ collapseDocID = collapseValues.advance(contextDoc);
+ }
+
+ int collapseKey;
+ if (collapseDocID == contextDoc) {
+ collapseKey = (int) collapseValues.longValue();
+ } else {
+ collapseKey = 0;
+ }
+
int globalDoc = contextDoc+this.docBase;
- int collapseKey = (int)this.collapseValues.get(contextDoc);
collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);
}
@@ -1130,7 +1211,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int contextDoc = globalDoc-currentDocBase;
if(this.needsScores){
- int collapseValue = (int)collapseValues.get(contextDoc);
+ int collapseDocID = collapseValues.docID();
+ if (collapseDocID < contextDoc) {
+ collapseDocID = collapseValues.advance(contextDoc);
+ }
+
+ int collapseValue;
+ if (collapseDocID == contextDoc) {
+ collapseValue = (int) collapseValues.longValue();
+ } else {
+ collapseValue = 0;
+ }
+
if(collapseValue != nullValue) {
int pointer = cmap.get(collapseValue);
dummy.score = scores[pointer];
@@ -1166,7 +1258,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
IntIntHashMap boostDocs,
SolrIndexSearcher searcher) throws IOException {
- SortedDocValues docValues = null;
+ DocValuesProducer docValuesProducer = null;
FunctionQuery funcQuery = null;
FieldType collapseFieldType = searcher.getSchema().getField(collapseField).getType();
@@ -1183,9 +1275,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
Map<String, UninvertingReader.Type> mapping = new HashMap();
mapping.put(collapseField, UninvertingReader.Type.SORTED);
UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getLeafReader(), collapseField), mapping);
- docValues = uninvertingReader.getSortedDocValues(collapseField);
+ docValuesProducer = new EmptyDocValuesProducer() {
+ @Override
+ public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
+ return uninvertingReader.getSortedDocValues(collapseField);
+ }
+ };
} else {
- docValues = DocValues.getSorted(searcher.getLeafReader(), collapseField);
+ docValuesProducer = new EmptyDocValuesProducer() {
+ @Override
+ public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
+ return DocValues.getSorted(searcher.getLeafReader(), collapseField);
+ }
+ };
}
} else {
if(HINT_TOP_FC.equals(hint)) {
@@ -1220,7 +1322,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if (collapseFieldType instanceof StrField) {
- return new OrdScoreCollector(maxDoc, leafCount, docValues, nullPolicy, boostDocs);
+ return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs);
} else if (collapseFieldType instanceof TrieIntField ||
collapseFieldType instanceof TrieFloatField) {
@@ -1251,7 +1353,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return new OrdFieldValueCollector(maxDoc,
leafCount,
- docValues,
+ docValuesProducer,
nullPolicy,
groupHeadSelector,
sortSpec,
@@ -1491,8 +1593,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- int currentVal = (int) minMaxValues.get(contextDoc);
+ int valuesDocID = minMaxValues.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = minMaxValues.advance(contextDoc);
+ }
+ int currentVal;
+ if (valuesDocID == contextDoc) {
+ currentVal = (int) minMaxValues.longValue();
+ } else {
+ currentVal = 0;
+ }
+
if(ord > -1) {
if(comp.test(currentVal, ordVals[ord])) {
ords[ord] = globalDoc;
@@ -1573,7 +1685,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- int currentMinMax = (int) minMaxValues.get(contextDoc);
+ int valuesDocID = minMaxValues.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = minMaxValues.advance(contextDoc);
+ }
+
+ int currentMinMax;
+ if (valuesDocID == contextDoc) {
+ currentMinMax = (int) minMaxValues.longValue();
+ } else {
+ currentMinMax = 0;
+ }
+
float currentVal = Float.intBitsToFloat(currentMinMax);
if(ord > -1) {
@@ -1655,7 +1778,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- long currentVal = minMaxVals.get(contextDoc);
+ int valuesDocID = minMaxVals.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = minMaxVals.advance(contextDoc);
+ }
+
+ long currentVal;
+ if (valuesDocID == contextDoc) {
+ currentVal = minMaxVals.longValue();
+ } else {
+ currentVal = 0;
+ }
+
if(ord > -1) {
if(comp.test(currentVal, ordVals[ord])) {
ords[ord] = globalDoc;
@@ -2051,7 +2185,17 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- int currentVal = (int) minMaxVals.get(contextDoc);
+ int valuesDocID = minMaxVals.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = minMaxVals.advance(contextDoc);
+ }
+
+ int currentVal;
+ if (valuesDocID == contextDoc) {
+ currentVal = (int) minMaxVals.longValue();
+ } else {
+ currentVal = 0;
+ }
if(collapseKey != nullValue) {
final int idx;
@@ -2153,7 +2297,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- int minMaxVal = (int) minMaxVals.get(contextDoc);
+ int valuesDocID = minMaxVals.docID();
+ if (valuesDocID < contextDoc) {
+ valuesDocID = minMaxVals.advance(contextDoc);
+ }
+
+ int minMaxVal;
+ if (valuesDocID == contextDoc) {
+ minMaxVal = (int) minMaxVals.longValue();
+ } else {
+ minMaxVal = 0;
+ }
+
float currentVal = Float.intBitsToFloat(minMaxVal);
if(collapseKey != nullValue) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
index 33a446a..df493f0 100644
--- a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java
@@ -268,7 +268,7 @@ public class HashQParserPlugin extends QParserPlugin {
private interface HashKey {
public void setNextReader(LeafReaderContext reader) throws IOException;
- public long hashCode(int doc);
+ public long hashCode(int doc) throws IOException;
}
private class BytesHash implements HashKey {
@@ -287,8 +287,16 @@ public class HashQParserPlugin extends QParserPlugin {
values = context.reader().getSortedDocValues(field);
}
- public long hashCode(int doc) {
- BytesRef ref = values.get(doc);
+ public long hashCode(int doc) throws IOException {
+ if (doc > values.docID()) {
+ values.advance(doc);
+ }
+ BytesRef ref;
+ if (doc == values.docID()) {
+ ref = values.binaryValue();
+ } else {
+ ref = null;
+ }
this.fieldType.indexedToReadable(ref, charsRefBuilder);
CharsRef charsRef = charsRefBuilder.get();
return charsRef.hashCode();
@@ -308,8 +316,17 @@ public class HashQParserPlugin extends QParserPlugin {
values = context.reader().getNumericDocValues(field);
}
- public long hashCode(int doc) {
- long l = values.get(doc);
+ public long hashCode(int doc) throws IOException {
+ int valuesDocID = values.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = values.advance(doc);
+ }
+ long l;
+ if (valuesDocID == doc) {
+ l = values.longValue();
+ } else {
+ l = 0;
+ }
return Longs.hashCode(l);
}
}
@@ -346,7 +363,7 @@ public class HashQParserPlugin extends QParserPlugin {
key4.setNextReader(context);
}
- public long hashCode(int doc) {
+ public long hashCode(int doc) throws IOException {
return key1.hashCode(doc)+key2.hashCode(doc)+key3.hashCode(doc)+key4.hashCode(doc);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java
index ee8cf67..bccbe06 100644
--- a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java
@@ -126,7 +126,18 @@ public class IGainTermsQParserPlugin extends QParserPlugin {
public void collect(int doc) throws IOException {
super.collect(doc);
++count;
- if (leafOutcomeValue.get(doc) == positiveLabel) {
+ int valuesDocID = leafOutcomeValue.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = leafOutcomeValue.advance(doc);
+ }
+ int value;
+ if (valuesDocID == doc) {
+ value = (int) leafOutcomeValue.longValue();
+ } else {
+ value = 0;
+ }
+
+ if (value == positiveLabel) {
positiveSet.set(context.docBase + doc);
numPositiveDocs++;
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 0dec53e..000d957 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -38,13 +38,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.collect.Iterables;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.document.LazyDocument;
import org.apache.lucene.index.*;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.StoredFieldVisitor.Status;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -53,13 +51,13 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrDocumentBase;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
@@ -84,6 +82,10 @@ import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.collect.Iterables;
+
/**
* SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}.
*
@@ -761,27 +763,30 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
continue;
}
- if (!DocValues.getDocsWithField(leafReader, fieldName).get(docid)) {
- continue;
- }
-
if (schemaField.multiValued()) {
final SortedSetDocValues values = reader.getSortedSetDocValues(fieldName);
if (values != null && values.getValueCount() > 0) {
- values.setDocument(docid);
- final List<Object> outValues = new LinkedList<Object>();
- for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
- final BytesRef value = values.lookupOrd(ord);
- outValues.add(schemaField.getType().toObject(schemaField, value));
+ if (values.advance(docid) == docid) {
+ final List<Object> outValues = new LinkedList<Object>();
+ for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+ final BytesRef value = values.lookupOrd(ord);
+ outValues.add(schemaField.getType().toObject(schemaField, value));
+ }
+ assert outValues.size() > 0;
+ doc.addField(fieldName, outValues);
}
- if (outValues.size() > 0) doc.addField(fieldName, outValues);
}
} else {
final DocValuesType dvType = fieldInfos.fieldInfo(fieldName).getDocValuesType();
switch (dvType) {
case NUMERIC:
final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName);
- Long val = ndv.get(docid);
+ Long val;
+ if (ndv.advance(docid) == docid) {
+ val = ndv.longValue();
+ } else {
+ continue;
+ }
Object newVal = val;
if (schemaField.getType() instanceof TrieIntField) {
newVal = val.intValue();
@@ -798,18 +803,23 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
break;
case BINARY:
BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName);
- doc.addField(fieldName, bdv.get(docid));
+ BytesRef value;
+ if (bdv.advance(docid) == docid) {
+ value = BytesRef.deepCopyOf(bdv.binaryValue());
+ } else {
+ continue;
+ }
+ doc.addField(fieldName, value);
break;
case SORTED:
SortedDocValues sdv = leafReader.getSortedDocValues(fieldName);
- int ord = sdv.getOrd(docid);
- if (ord >= 0) {
+ if (sdv.advance(docid) == docid) {
+ final BytesRef bRef = sdv.binaryValue();
// Special handling for Boolean fields since they're stored as 'T' and 'F'.
if (schemaField.getType() instanceof BoolField) {
- final BytesRef bRef = sdv.lookupOrd(ord);
doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef));
} else {
- doc.addField(fieldName, sdv.get(docid).utf8ToString());
+ doc.addField(fieldName, bRef.utf8ToString());
}
}
break;
@@ -818,8 +828,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
case SORTED_SET:
throw new AssertionError("SORTED_SET fields should be multi-valued!");
case NONE:
- // Shouldn't happen since we check that the document has a DocValues field.
- throw new AssertionError("Document does not have a DocValues field with the name '" + fieldName + "'!");
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java
index 0ca9f72..96f869f 100644
--- a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java
@@ -151,8 +151,17 @@ public class TextLogisticRegressionQParserPlugin extends QParserPlugin {
}
public void collect(int doc) throws IOException{
+ int valuesDocID = leafOutcomeValue.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = leafOutcomeValue.advance(valuesDocID);
+ }
+ int outcome;
+ if (valuesDocID == doc) {
+ outcome = (int) leafOutcomeValue.longValue();
+ } else {
+ outcome = 0;
+ }
- int outcome = (int) leafOutcomeValue.get(doc);
outcome = trainingParams.positiveLabel == outcome? 1 : 0;
if (outcome == 1) {
positiveDocsSet.set(context.docBase + doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
index dd0db96..c2b8a5d 100644
--- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
@@ -219,7 +219,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
return "mod";
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.floatVal(doc) % bVals.floatVal(doc);
}
};
@@ -248,7 +248,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
}
@Override
- protected float func(int doc, FunctionValues vals) {
+ protected float func(int doc, FunctionValues vals) throws IOException {
return Math.abs(vals.floatVal(doc));
}
};
@@ -290,7 +290,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.floatVal(doc) - bVals.floatVal(doc);
}
};
@@ -450,133 +450,133 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
addParser(new DoubleParser("rad") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return vals.doubleVal(doc) * DistanceUtils.DEGREES_TO_RADIANS;
}
});
addParser(new DoubleParser("deg") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return vals.doubleVal(doc) * DistanceUtils.RADIANS_TO_DEGREES;
}
});
addParser(new DoubleParser("sqrt") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.sqrt(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("cbrt") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.cbrt(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("log") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.log10(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("ln") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.log(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("exp") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.exp(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("sin") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.sin(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("cos") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.cos(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("tan") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.tan(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("asin") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.asin(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("acos") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.acos(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("atan") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.atan(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("sinh") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.sinh(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("cosh") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.cosh(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("tanh") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.tanh(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("ceil") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.ceil(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("floor") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.floor(vals.doubleVal(doc));
}
});
addParser(new DoubleParser("rint") {
@Override
- public double func(int doc, FunctionValues vals) {
+ public double func(int doc, FunctionValues vals) throws IOException {
return Math.rint(vals.doubleVal(doc));
}
});
addParser(new Double2Parser("pow") {
@Override
- public double func(int doc, FunctionValues a, FunctionValues b) {
+ public double func(int doc, FunctionValues a, FunctionValues b) throws IOException {
return Math.pow(a.doubleVal(doc), b.doubleVal(doc));
}
});
addParser(new Double2Parser("hypot") {
@Override
- public double func(int doc, FunctionValues a, FunctionValues b) {
+ public double func(int doc, FunctionValues a, FunctionValues b) throws IOException {
return Math.hypot(a.doubleVal(doc), b.doubleVal(doc));
}
});
addParser(new Double2Parser("atan2") {
@Override
- public double func(int doc, FunctionValues a, FunctionValues b) {
+ public double func(int doc, FunctionValues a, FunctionValues b) throws IOException {
return Math.atan2(a.doubleVal(doc), b.doubleVal(doc));
}
});
@@ -727,7 +727,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
return "exists";
}
@Override
- protected boolean func(int doc, FunctionValues vals) {
+ protected boolean func(int doc, FunctionValues vals) throws IOException {
return vals.exists(doc);
}
};
@@ -740,7 +740,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
ValueSource vs = fp.parseValueSource();
return new SimpleBoolFunction(vs) {
@Override
- protected boolean func(int doc, FunctionValues vals) {
+ protected boolean func(int doc, FunctionValues vals) throws IOException {
return !vals.boolVal(doc);
}
@Override
@@ -762,7 +762,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
return "and";
}
@Override
- protected boolean func(int doc, FunctionValues[] vals) {
+ protected boolean func(int doc, FunctionValues[] vals) throws IOException {
for (FunctionValues dv : vals)
if (!dv.boolVal(doc)) return false;
return true;
@@ -781,7 +781,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
return "or";
}
@Override
- protected boolean func(int doc, FunctionValues[] vals) {
+ protected boolean func(int doc, FunctionValues[] vals) throws IOException {
for (FunctionValues dv : vals)
if (dv.boolVal(doc)) return true;
return false;
@@ -800,7 +800,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
return "xor";
}
@Override
- protected boolean func(int doc, FunctionValues[] vals) {
+ protected boolean func(int doc, FunctionValues[] vals) throws IOException {
int nTrue=0, nFalse=0;
for (FunctionValues dv : vals) {
if (dv.boolVal(doc)) nTrue++;
@@ -1121,7 +1121,7 @@ class DateValueSourceParser extends ValueSourceParser {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return ms1 - bVals.longVal(doc);
}
};
@@ -1135,7 +1135,7 @@ class DateValueSourceParser extends ValueSourceParser {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.longVal(doc) - ms2;
}
};
@@ -1149,7 +1149,7 @@ class DateValueSourceParser extends ValueSourceParser {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.longVal(doc) - bVals.longVal(doc);
}
};
@@ -1267,7 +1267,7 @@ abstract class DoubleParser extends NamedParser {
super(name);
}
- public abstract double func(int doc, FunctionValues vals);
+ public abstract double func(int doc, FunctionValues vals) throws IOException;
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
@@ -1289,11 +1289,11 @@ abstract class DoubleParser extends NamedParser {
final FunctionValues vals = source.getValues(context, readerContext);
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return func(doc, vals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + vals.toString(doc) + ')';
}
};
@@ -1307,7 +1307,7 @@ abstract class Double2Parser extends NamedParser {
super(name);
}
- public abstract double func(int doc, FunctionValues a, FunctionValues b);
+ public abstract double func(int doc, FunctionValues a, FunctionValues b) throws IOException;
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
@@ -1338,11 +1338,11 @@ abstract class Double2Parser extends NamedParser {
final FunctionValues bVals = b.getValues(context, readerContext);
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return func(doc, aVals, bVals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java
index 1ef4284..fb60945 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java
@@ -57,8 +57,9 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray {
ordinalMap = ((MultiDocValues.MultiSortedSetDocValues)si).mapping;
}
} else {
+ // multi-valued view
SortedDocValues single = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null);
- si = DocValues.singleton(single); // multi-valued view
+ si = DocValues.singleton(single);
if (single instanceof MultiDocValues.MultiSortedDocValues) {
ordinalMap = ((MultiDocValues.MultiSortedDocValues)single).mapping;
}
@@ -185,7 +186,14 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- counts[ singleDv.getOrd(doc) + 1 ]++;
+ if (doc > singleDv.docID()) {
+ singleDv.advance(doc);
+ }
+ if (doc == singleDv.docID()) {
+ counts[ singleDv.ordValue() + 1 ]++;
+ } else {
+ counts[ 0 ]++;
+ }
}
for (int i=1; i<segMax; i++) {
@@ -203,11 +211,15 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- multiDv.setDocument(doc);
- for(;;) {
- int segOrd = (int)multiDv.nextOrd();
- if (segOrd < 0) break;
- counts[segOrd]++;
+ if (doc > multiDv.docID()) {
+ multiDv.advance(doc);
+ }
+ if (doc == multiDv.docID()) {
+ for(;;) {
+ int segOrd = (int)multiDv.nextOrd();
+ if (segOrd < 0) break;
+ counts[segOrd]++;
+ }
}
}
@@ -235,30 +247,42 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray {
private void collectDocs(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- int segOrd = singleDv.getOrd(doc);
- if (segOrd < 0) continue;
- collect(doc, segOrd, toGlobal);
+ if (doc > singleDv.docID()) {
+ singleDv.advance(doc);
+ }
+ if (doc == singleDv.docID()) {
+ int segOrd = singleDv.ordValue();
+ collect(doc, segOrd, toGlobal);
+ }
}
}
private void collectCounts(SortedDocValues singleDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- int segOrd = singleDv.getOrd(doc);
- if (segOrd < 0) continue;
- int ord = (int)toGlobal.get(segOrd);
- countAcc.incrementCount(ord, 1);
+ if (doc > singleDv.docID()) {
+ singleDv.advance(doc);
+ }
+ if (doc == singleDv.docID()) {
+ int segOrd = singleDv.ordValue();
+ int ord = (int)toGlobal.get(segOrd);
+ countAcc.incrementCount(ord, 1);
+ }
}
}
private void collectDocs(SortedSetDocValues multiDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- multiDv.setDocument(doc);
- for(;;) {
- int segOrd = (int)multiDv.nextOrd();
- if (segOrd < 0) break;
- collect(doc, segOrd, toGlobal);
+ if (doc > multiDv.docID()) {
+ multiDv.advance(doc);
+ }
+ if (doc == multiDv.docID()) {
+ for(;;) {
+ int segOrd = (int)multiDv.nextOrd();
+ if (segOrd < 0) break;
+ collect(doc, segOrd, toGlobal);
+ }
}
}
}
@@ -266,12 +290,16 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray {
private void collectCounts(SortedSetDocValues multiDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- multiDv.setDocument(doc);
- for(;;) {
- int segOrd = (int)multiDv.nextOrd();
- if (segOrd < 0) break;
- int ord = (int)toGlobal.get(segOrd);
- countAcc.incrementCount(ord, 1);
+ if (doc > multiDv.docID()) {
+ multiDv.advance(doc);
+ }
+ if (doc == multiDv.docID()) {
+ for(;;) {
+ int segOrd = (int)multiDv.nextOrd();
+ if (segOrd < 0) break;
+ int ord = (int)toGlobal.get(segOrd);
+ countAcc.incrementCount(ord, 1);
+ }
}
}
}
[02/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
index fb93417..ffed5f4 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
@@ -29,7 +29,6 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.SolrException;
@@ -353,9 +352,11 @@ class FacetFieldProcessorByHashDV extends FacetFieldProcessor {
@Override
public void collect(int segDoc) throws IOException {
- long ord = docValues.getOrd(segDoc);
- if (ord != -1) {
- long val = toGlobal.get(ord);
+ if (segDoc > docValues.docID()) {
+ docValues.advance(segDoc);
+ }
+ if (segDoc == docValues.docID()) {
+ long val = toGlobal.get(docValues.ordValue());
collectValFirstPhase(segDoc, val);
}
}
@@ -366,7 +367,6 @@ class FacetFieldProcessorByHashDV extends FacetFieldProcessor {
// TODO support SortedNumericDocValues
DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {
NumericDocValues values = null; //NN
- Bits docsWithField = null; //NN
@Override public boolean needsScores() { return false; }
@@ -374,14 +374,15 @@ class FacetFieldProcessorByHashDV extends FacetFieldProcessor {
protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
setNextReaderFirstPhase(ctx);
values = DocValues.getNumeric(ctx.reader(), sf.getName());
- docsWithField = DocValues.getDocsWithField(ctx.reader(), sf.getName());
}
@Override
public void collect(int segDoc) throws IOException {
- long val = values.get(segDoc);
- if (val != 0 || docsWithField.get(segDoc)) {
- collectValFirstPhase(segDoc, val);
+ if (segDoc > values.docID()) {
+ values.advance(segDoc);
+ }
+ if (segDoc == values.docID()) {
+ collectValFirstPhase(segDoc, values.longValue());
}
}
});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
index 89e2386..005e9fc 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
@@ -22,7 +22,6 @@ import org.apache.solr.util.hll.HLLType;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
@@ -112,7 +111,6 @@ public class HLLAgg extends StrAggValueSource {
SchemaField sf;
HLL[] sets;
NumericDocValues values;
- Bits exists;
public NumericAcc(FacetContext fcontext, String field, int numSlots) throws IOException {
super(fcontext);
@@ -133,15 +131,20 @@ public class HLLAgg extends StrAggValueSource {
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
- exists = DocValues.getDocsWithField(readerContext.reader(), sf.getName());
}
@Override
public void collect(int doc, int slot) throws IOException {
- long val = values.get(doc);
- if (val == 0 && !exists.get(doc)) {
+ int valuesDocID = values.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = values.advance(doc);
+ }
+ if (valuesDocID > doc) {
return;
}
+ assert valuesDocID == doc;
+
+ long val = values.longValue();
long hash = Hash.fmix64(val);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
index a1f44f0..7f063b2 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
@@ -109,7 +109,7 @@ public class PercentileAgg extends SimpleAggValueSource {
digests = new AVLTreeDigest[numSlots];
}
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
if (!values.exists(doc)) return;
double val = values.doubleVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
index de1636e..4f4851b 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
@@ -265,7 +265,7 @@ class SumSlotAcc extends DoubleFuncSlotAcc {
super(values, fcontext, numSlots);
}
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
double val = values.doubleVal(doc); // todo: worth trying to share this value across multiple stats that need it?
result[slotNum] += val;
}
@@ -277,7 +277,7 @@ class SumsqSlotAcc extends DoubleFuncSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
double val = values.doubleVal(doc);
val = val * val;
result[slotNum] += val;
@@ -292,7 +292,7 @@ class MinSlotAcc extends DoubleFuncSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
double val = values.doubleVal(doc);
if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
@@ -309,7 +309,7 @@ class MaxSlotAcc extends DoubleFuncSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
double val = values.doubleVal(doc);
if (val == 0 && !values.exists(doc)) return; // depend on fact that non existing values return 0 for func query
@@ -339,7 +339,7 @@ class AvgSlotAcc extends DoubleFuncSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
+ public void collect(int doc, int slotNum) throws IOException {
double val = values.doubleVal(doc);
if (val != 0 || values.exists(doc)) {
result[slotNum] += val;
@@ -466,4 +466,4 @@ class SortSlotAcc extends SlotAcc {
// sort slot only works with direct-mapped accumulators
throw new UnsupportedOperationException();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
index 341bdaf..a18eb0f 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
@@ -25,7 +25,6 @@ import java.util.Set;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
@@ -192,7 +191,6 @@ public class UniqueAgg extends StrAggValueSource {
SchemaField sf;
LongSet[] sets;
NumericDocValues values;
- Bits exists;
public NumericAcc(FacetContext fcontext, String field, int numSlots) throws IOException {
super(fcontext);
@@ -213,15 +211,19 @@ public class UniqueAgg extends StrAggValueSource {
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
- exists = DocValues.getDocsWithField(readerContext.reader(), sf.getName());
}
@Override
public void collect(int doc, int slot) throws IOException {
- long val = values.get(doc);
- if (val == 0 && !exists.get(doc)) {
+ int valuesDocID = values.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = values.advance(doc);
+ }
+ if (valuesDocID > doc) {
+ // missing
return;
}
+ long val = values.longValue();
LongSet set = sets[slot];
if (set == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
index 4c29753..c52df36 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
@@ -66,21 +66,26 @@ class UniqueMultiDvSlotAcc extends UniqueSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
- subDv.setDocument(doc);
- int segOrd = (int) subDv.nextOrd();
- if (segOrd < 0) return;
-
- FixedBitSet bits = arr[slotNum];
- if (bits == null) {
- bits = new FixedBitSet(nTerms);
- arr[slotNum] = bits;
+ public void collect(int doc, int slotNum) throws IOException {
+ if (doc > subDv.docID()) {
+ subDv.advance(doc);
}
+ if (doc == subDv.docID()) {
+
+ int segOrd = (int) subDv.nextOrd();
+ assert segOrd >= 0;
+
+ FixedBitSet bits = arr[slotNum];
+ if (bits == null) {
+ bits = new FixedBitSet(nTerms);
+ arr[slotNum] = bits;
+ }
- do {
- int ord = toGlobal == null ? segOrd : (int) toGlobal.get(segOrd);
- bits.set(ord);
- segOrd = (int) subDv.nextOrd();
- } while (segOrd >= 0);
+ do {
+ int ord = toGlobal == null ? segOrd : (int) toGlobal.get(segOrd);
+ bits.set(ord);
+ segOrd = (int) subDv.nextOrd();
+ } while (segOrd >= 0);
+ }
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
index c67fd47..870b894 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
@@ -66,16 +66,20 @@ class UniqueSinglevaluedSlotAcc extends UniqueSlotAcc {
}
@Override
- public void collect(int doc, int slotNum) {
- int segOrd = subDv.getOrd(doc);
- if (segOrd < 0) return; // -1 means missing
- int ord = toGlobal==null ? segOrd : (int)toGlobal.get(segOrd);
+ public void collect(int doc, int slotNum) throws IOException {
+ if (doc > subDv.docID()) {
+ subDv.advance(doc);
+ }
+ if (doc == subDv.docID()) {
+ int segOrd = subDv.ordValue();
+ int ord = toGlobal==null ? segOrd : (int)toGlobal.get(segOrd);
- FixedBitSet bits = arr[slotNum];
- if (bits == null) {
- bits = new FixedBitSet(nTerms);
- arr[slotNum] = bits;
+ FixedBitSet bits = arr[slotNum];
+ if (bits == null) {
+ bits = new FixedBitSet(nTerms);
+ arr[slotNum] = bits;
+ }
+ bits.set(ord);
}
- bits.set(ord);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
index 3b63bd9..40d9948 100644
--- a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
+++ b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
@@ -97,16 +97,33 @@ public class OrdFieldSource extends ValueSource {
// if it's e.g. tokenized/multivalued, emulate old behavior of single-valued fc
final SortedDocValues sindex = SortedSetSelector.wrap(DocValues.getSortedSet(r, field), SortedSetSelector.Type.MIN);
return new IntDocValues(this) {
+
+ private int lastDocID;
+
+ private int getOrdForDoc(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new IllegalArgumentException("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > sindex.docID()) {
+ sindex.advance(docID);
+ }
+ if (docID == sindex.docID()) {
+ return sindex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
protected String toTerm(String readableValue) {
return readableValue;
}
@Override
- public int intVal(int doc) {
- return sindex.getOrd(doc+off);
+ public int intVal(int doc) throws IOException {
+ return getOrdForDoc(doc+off);
}
@Override
- public int ordVal(int doc) {
- return sindex.getOrd(doc+off);
+ public int ordVal(int doc) throws IOException {
+ return getOrdForDoc(doc+off);
}
@Override
public int numOrd() {
@@ -114,8 +131,8 @@ public class OrdFieldSource extends ValueSource {
}
@Override
- public boolean exists(int doc) {
- return sindex.getOrd(doc+off) != 0;
+ public boolean exists(int doc) throws IOException {
+ return getOrdForDoc(doc+off) != 0;
}
@Override
@@ -129,8 +146,8 @@ public class OrdFieldSource extends ValueSource {
}
@Override
- public void fillValue(int doc) {
- mval.value = sindex.getOrd(doc);
+ public void fillValue(int doc) throws IOException {
+ mval.value = getOrdForDoc(doc);
mval.exists = mval.value!=0;
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
index abc9f95..4417000 100644
--- a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
+++ b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
@@ -99,9 +99,16 @@ public class ReverseOrdFieldSource extends ValueSource {
final int end = sindex.getValueCount();
return new IntDocValues(this) {
- @Override
- public int intVal(int doc) {
- return (end - sindex.getOrd(doc+off) - 1);
+ @Override
+ public int intVal(int doc) throws IOException {
+ if (doc+off > sindex.docID()) {
+ sindex.advance(doc+off);
+ }
+ if (doc+off == sindex.docID()) {
+ return (end - sindex.ordValue() - 1);
+ } else {
+ return end;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java b/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java
index c994fbb..64be371 100644
--- a/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java
@@ -17,6 +17,8 @@
package org.apache.solr.search.function;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
@@ -41,7 +43,7 @@ public class SolrComparisonBoolFunction extends ComparisonBoolFunction {
}
@Override
- public boolean compare(int doc, FunctionValues lhs, FunctionValues rhs) {
+ public boolean compare(int doc, FunctionValues lhs, FunctionValues rhs) throws IOException {
// TODO consider a separate FunctionValues impl, one for Long, one for Double
// performs the safest possible numeric comparison, if both lhs and rhs are Longs, then
// we perform a Long comparison to avoid the issues with precision when casting to doubles
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
index b00f2fd..4e38843 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
@@ -53,12 +53,12 @@ public class GeohashFunction extends ValueSource {
return new FunctionValues() {
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return GeohashUtils.encodeLatLon(latDV.doubleVal(doc), lonDV.doubleVal(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
sb.append(latDV.toString(doc)).append(',').append(lonDV.toString(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
index 915db33..40c7aa7 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
@@ -65,11 +65,11 @@ public class GeohashHaversineFunction extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return distance(doc, gh1DV, gh2DV);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
sb.append(gh1DV.toString(doc)).append(',').append(gh2DV.toString(doc));
@@ -79,7 +79,7 @@ public class GeohashHaversineFunction extends ValueSource {
};
}
- protected double distance(int doc, FunctionValues gh1DV, FunctionValues gh2DV) {
+ protected double distance(int doc, FunctionValues gh1DV, FunctionValues gh2DV) throws IOException {
double result = 0;
String h1 = gh1DV.strVal(doc);
String h2 = gh2DV.strVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
index 4b68f5c..e489ff4 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
@@ -65,7 +65,7 @@ public class HaversineConstFunction extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
double latRad = latVals.doubleVal(doc) * DEGREES_TO_RADIANS;
double lonRad = lonVals.doubleVal(doc) * DEGREES_TO_RADIANS;
double diffX = latCenterRad - latRad;
@@ -77,7 +77,7 @@ public class HaversineConstFunction extends ValueSource {
return (EARTH_MEAN_DIAMETER * Math.atan2(Math.sqrt(h), Math.sqrt(1 - h)));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + latVals.toString(doc) + ',' + lonVals.toString(doc) + ',' + latCenter + ',' + lonCenter + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
index 20ed5a7..1697f2b 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
@@ -67,7 +67,7 @@ public class HaversineFunction extends ValueSource {
* @param doc The doc to score
* @return The haversine distance formula
*/
- protected double distance(int doc, FunctionValues p1DV, FunctionValues p2DV) {
+ protected double distance(int doc, FunctionValues p1DV, FunctionValues p2DV) throws IOException {
double[] p1D = new double[2];
double[] p2D = new double[2];
@@ -99,11 +99,11 @@ public class HaversineFunction extends ValueSource {
final FunctionValues vals2 = p2.getValues(context, readerContext);
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return distance(doc, vals1, vals2);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
sb.append(vals1.toString(doc)).append(',').append(vals2.toString(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java
index 75c7fc0..381a526 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java
@@ -15,6 +15,8 @@
* limitations under the License.
*/
package org.apache.solr.search.function.distance;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.valuesource.MultiValueSource;
@@ -40,7 +42,7 @@ public class SquaredEuclideanFunction extends VectorDistanceFunction {
* @param doc The doc to score
*/
@Override
- protected double distance(int doc, FunctionValues dv1, FunctionValues dv2) {
+ protected double distance(int doc, FunctionValues dv1, FunctionValues dv2) throws IOException {
double[] vals1 = new double[source1.dimension()];
double[] vals2 = new double[source1.dimension()];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
index ed94e31..8bf9f1a 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
@@ -49,7 +49,7 @@ public class StringDistanceFunction extends ValueSource {
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
String s1 = str1DV.strVal(doc);
String s2 = str2DV.strVal(doc);
if (null == s1 || null == s2) {
@@ -60,12 +60,12 @@ public class StringDistanceFunction extends ValueSource {
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return str1DV.exists(doc) && str2DV.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append("strdist").append('(');
sb.append(str1DV.toString(doc)).append(',').append(str2DV.toString(doc))
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
index f988882..474ece3 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
@@ -67,7 +67,7 @@ public class VectorDistanceFunction extends ValueSource {
* @param dv2 The values from the second MultiValueSource
* @return The distance
*/
- protected double distance(int doc, FunctionValues dv1, FunctionValues dv2) {
+ protected double distance(int doc, FunctionValues dv1, FunctionValues dv2) throws IOException {
//Handle some special cases:
double[] vals1 = new double[source1.dimension()];
double[] vals2 = new double[source1.dimension()];
@@ -159,12 +159,12 @@ public class VectorDistanceFunction extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return distance(doc, vals1, vals2);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(').append(power).append(',');
boolean firstTime = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/join/BlockJoinFieldFacetAccumulator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFieldFacetAccumulator.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFieldFacetAccumulator.java
index 286cd45..dee1f60 100644
--- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinFieldFacetAccumulator.java
+++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinFieldFacetAccumulator.java
@@ -21,8 +21,8 @@ import java.util.Arrays;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.ArrayUtil;
@@ -66,10 +66,12 @@ class BlockJoinFieldFacetAccumulator {
}
} else {
SortedDocValues single = searcher.getLeafReader().getSortedDocValues(fieldName);
- topSSDV = single == null ? null : DocValues.singleton(single);// npe friendly code
if (single instanceof MultiDocValues.MultiSortedDocValues) {
ordinalMap = ((MultiDocValues.MultiSortedDocValues) single).mapping;
}
+ if (single != null) {
+ topSSDV = DocValues.singleton(single);
+ }
}
}
@@ -135,18 +137,31 @@ class BlockJoinFieldFacetAccumulator {
// some codecs may optimize SORTED_SET storage for single-valued fields
for (iter.reset(); iter.hasNext(); ) {
final int docNum = iter.nextDoc();
- int term = segmentSDV.getOrd(docNum);
+ if (docNum > segmentSDV.docID()) {
+ segmentSDV.advance(docNum);
+ }
+
+ int term;
+ if (docNum == segmentSDV.docID()) {
+ term = segmentSDV.ordValue();
+ } else {
+ term = -1;
+ }
accumulateTermOrd(term, iter.getAggKey());
//System.out.println("doc# "+docNum+" "+fieldName+" term# "+term+" tick "+Long.toHexString(segmentAccums[1+term]));
}
} else {
for (iter.reset(); iter.hasNext(); ) {
final int docNum = iter.nextDoc();
- segmentSSDV.setDocument(docNum);
- int term = (int) segmentSSDV.nextOrd();
- do { // absent values are designated by term=-1, first iteration counts [0] as "missing", and exit, otherwise it spins
- accumulateTermOrd(term, iter.getAggKey());
- } while (term>=0 && (term = (int) segmentSSDV.nextOrd()) >= 0);
+ if (docNum > segmentSSDV.docID()) {
+ segmentSSDV.advance(docNum);
+ }
+ if (docNum == segmentSSDV.docID()) {
+ int term = (int) segmentSSDV.nextOrd();
+ do { // absent values are designated by term=-1, first iteration counts [0] as "missing", and exit, otherwise it spins
+ accumulateTermOrd(term, iter.getAggKey());
+ } while (term>=0 && (term = (int) segmentSSDV.nextOrd()) >= 0);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java b/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java
index 33a03f4..377f71b 100644
--- a/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java
+++ b/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java
@@ -89,14 +89,18 @@ class GraphTermsCollector extends SimpleCollector implements Collector {
private void addEdgeIdsToResult(int doc) throws IOException {
// set the doc to pull the edges ids for.
- docTermOrds.setDocument(doc);
- BytesRef edgeValue = new BytesRef();
- long ord;
- while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- // TODO: handle non string type fields.
- edgeValue = docTermOrds.lookupOrd(ord);
- // add the edge id to the collector terms.
- collectorTerms.add(edgeValue);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ BytesRef edgeValue = new BytesRef();
+ long ord;
+ while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ // TODO: handle non string type fields.
+ edgeValue = docTermOrds.lookupOrd(ord);
+ // add the edge id to the collector terms.
+ collectorTerms.add(edgeValue);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
index 4b60dba..fcc6974 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
@@ -27,6 +27,8 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LegacySortedSetDocValues;
+import org.apache.lucene.index.LegacySortedSetDocValuesWrapper;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
@@ -745,11 +747,11 @@ public class DocTermOrds implements Accountable {
if (isEmpty()) {
return DocValues.emptySortedSet();
} else {
- return new Iterator(reader);
+ return new LegacySortedSetDocValuesWrapper(new Iterator(reader), reader.maxDoc());
}
}
- private class Iterator extends SortedSetDocValues {
+ private class Iterator extends LegacySortedSetDocValues {
final LeafReader reader;
final TermsEnum te; // used internally for lookupOrd() and lookupTerm()
// currently we read 5 at a time (using the logic of the old iterator)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
index be08a60..ea8f6ea 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
@@ -20,9 +20,9 @@ import java.io.IOException;
import java.io.PrintStream;
import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.IndexReader; // javadocs
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -280,14 +280,11 @@ interface FieldCache {
* Computes long for string values. May be {@code null} if the
* requested field was indexed as {@link NumericDocValuesField} or
* {@link org.apache.lucene.legacy.LegacyLongField}.
- * @param setDocsWithField
- * If true then {@link #getDocsWithField} will also be computed and
- * stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException
* If any error occurs.
*/
- public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser, boolean setDocsWithField) throws IOException;
+ public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
@@ -295,19 +292,17 @@ interface FieldCache {
* method to retrieve the term (as a BytesRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
- * @param setDocsWithField If true then {@link #getDocsWithField} will
- * also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public BinaryDocValues getTerms(LeafReader reader, String field, boolean setDocsWithField) throws IOException;
+ public BinaryDocValues getTerms(LeafReader reader, String field) throws IOException;
- /** Expert: just like {@link #getTerms(org.apache.lucene.index.LeafReader,String,boolean)},
+ /** Expert: just like {@link #getTerms(org.apache.lucene.index.LeafReader,String)},
* but you can specify whether more RAM should be consumed in exchange for
* faster lookups (default is "true"). Note that the
* first call for a given reader and field "wins",
* subsequent calls will share the same cache entry. */
- public BinaryDocValues getTerms(LeafReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException;
+ public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
index e6a066d..ab14785 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
@@ -29,14 +29,14 @@ import java.util.WeakHashMap;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -134,7 +134,7 @@ class FieldCacheImpl implements FieldCache {
final Map<Object,Map<CacheKey,Accountable>> readerCache = new WeakHashMap<>();
- protected abstract Accountable createValue(LeafReader reader, CacheKey key, boolean setDocsWithField)
+ protected abstract Accountable createValue(LeafReader reader, CacheKey key)
throws IOException;
/** Remove this reader from the cache, if present. */
@@ -165,7 +165,7 @@ class FieldCacheImpl implements FieldCache {
}
}
- public Object get(LeafReader reader, CacheKey key, boolean setDocsWithField) throws IOException {
+ public Object get(LeafReader reader, CacheKey key) throws IOException {
Map<CacheKey,Accountable> innerCache;
Accountable value;
final Object readerKey = reader.getCoreCacheKey();
@@ -189,7 +189,7 @@ class FieldCacheImpl implements FieldCache {
synchronized (value) {
CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
- progress.value = createValue(reader, key, setDocsWithField);
+ progress.value = createValue(reader, key);
synchronized (readerCache) {
innerCache.put(key, progress.value);
}
@@ -272,31 +272,31 @@ class FieldCacheImpl implements FieldCache {
this.points = points;
}
- final void uninvert(LeafReader reader, String field, boolean setDocsWithField) throws IOException {
+ final void uninvert(LeafReader reader, String field) throws IOException {
if (points) {
- uninvertPoints(reader, field, setDocsWithField);
+ uninvertPoints(reader, field);
} else {
- uninvertPostings(reader, field, setDocsWithField);
+ uninvertPostings(reader, field);
}
}
- final void uninvertPoints(LeafReader reader, String field, boolean setDocsWithField) throws IOException {
+ final void uninvertPoints(LeafReader reader, String field) throws IOException {
final int maxDoc = reader.maxDoc();
PointValues values = reader.getPointValues();
assert values != null;
assert values.size(field) > 0;
-
- if (setDocsWithField) {
- final int docCount = values.getDocCount(field);
- assert docCount <= maxDoc;
- if (docCount == maxDoc) {
- // Fast case: all docs have this field:
- this.docsWithField = new Bits.MatchAllBits(maxDoc);
- setDocsWithField = false;
- }
+
+ final boolean setDocsWithField;
+ final int docCount = values.getDocCount(field);
+ assert docCount <= maxDoc;
+ if (docCount == maxDoc) {
+ // Fast case: all docs have this field:
+ this.docsWithField = new Bits.MatchAllBits(maxDoc);
+ setDocsWithField = false;
+ } else {
+ setDocsWithField = true;
}
- final boolean doDocsWithField = setDocsWithField;
BytesRef scratch = new BytesRef();
values.intersect(field, new IntersectVisitor() {
@Override
@@ -310,7 +310,7 @@ class FieldCacheImpl implements FieldCache {
scratch.length = packedValue.length;
visitTerm(scratch);
visitDoc(docID);
- if (doDocsWithField) {
+ if (setDocsWithField) {
if (docsWithField == null) {
// Lazy init
docsWithField = new FixedBitSet(maxDoc);
@@ -326,18 +326,19 @@ class FieldCacheImpl implements FieldCache {
});
}
- final void uninvertPostings(LeafReader reader, String field, boolean setDocsWithField) throws IOException {
+ final void uninvertPostings(LeafReader reader, String field) throws IOException {
final int maxDoc = reader.maxDoc();
Terms terms = reader.terms(field);
if (terms != null) {
- if (setDocsWithField) {
- final int termsDocCount = terms.getDocCount();
- assert termsDocCount <= maxDoc;
- if (termsDocCount == maxDoc) {
- // Fast case: all docs have this field:
- this.docsWithField = new Bits.MatchAllBits(maxDoc);
- setDocsWithField = false;
- }
+ final boolean setDocsWithField;
+ final int termsDocCount = terms.getDocCount();
+ assert termsDocCount <= maxDoc;
+ if (termsDocCount == maxDoc) {
+ // Fast case: all docs have this field:
+ this.docsWithField = new Bits.MatchAllBits(maxDoc);
+ setDocsWithField = false;
+ } else {
+ setDocsWithField = true;
}
final TermsEnum termsEnum = termsEnum(terms);
@@ -418,25 +419,25 @@ class FieldCacheImpl implements FieldCache {
public long minValue;
}
+ @Override
public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException {
final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
if (fieldInfo == null) {
// field does not exist or has no value
return new Bits.MatchNoBits(reader.maxDoc());
- } else if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
- return reader.getDocsWithField(field);
}
- if (parser instanceof PointParser) {
+ if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
+ // doc values case
+ } else if (parser instanceof PointParser) {
// points case
-
} else {
// postings case
if (fieldInfo.getIndexOptions() == IndexOptions.NONE) {
return new Bits.MatchNoBits(reader.maxDoc());
}
}
- BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser), false);
+ BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser));
return bitsEntry.bits;
}
@@ -464,16 +465,56 @@ class FieldCacheImpl implements FieldCache {
}
@Override
- protected BitsEntry createValue(LeafReader reader, CacheKey key, boolean setDocsWithField /* ignored */) throws IOException {
+ protected BitsEntry createValue(LeafReader reader, CacheKey key) throws IOException {
final String field = key.field;
final Parser parser = (Parser) key.custom;
- if (parser instanceof PointParser) {
+ FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+ if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
+ return createValueDocValues(reader, field);
+ } else if (parser instanceof PointParser) {
return createValuePoints(reader, field);
} else {
return createValuePostings(reader, field);
}
}
-
+
+ private BitsEntry createValueDocValues(LeafReader reader, String field) throws IOException {
+ FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+
+ DocValuesType dvType = fieldInfo.getDocValuesType();
+ DocIdSetIterator iterator;
+ switch(dvType) {
+ case NUMERIC:
+ iterator = reader.getNumericDocValues(field);
+ break;
+ case BINARY:
+ iterator = reader.getBinaryDocValues(field);
+ break;
+ case SORTED:
+ iterator = reader.getSortedDocValues(field);
+ break;
+ case SORTED_NUMERIC:
+ iterator = reader.getSortedNumericDocValues(field);
+ break;
+ case SORTED_SET:
+ iterator = reader.getSortedSetDocValues(field);
+ break;
+ default:
+ throw new AssertionError();
+ }
+
+ FixedBitSet bits = new FixedBitSet(reader.maxDoc());
+ while (true) {
+ int docID = iterator.nextDoc();
+ if (docID == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
+ bits.set(docID);
+ }
+
+ return new BitsEntry(bits);
+ }
+
private BitsEntry createValuePoints(LeafReader reader, String field) throws IOException {
final int maxDoc = reader.maxDoc();
PointValues values = reader.getPointValues();
@@ -500,7 +541,7 @@ class FieldCacheImpl implements FieldCache {
@Override
protected void visitDoc(int docID) {}
};
- u.uninvert(reader, field, true);
+ u.uninvert(reader, field);
return new BitsEntry(u.docsWithField);
}
@@ -553,16 +594,14 @@ class FieldCacheImpl implements FieldCache {
return new BitsEntry(res);
}
}
-
+
@Override
- public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser, boolean setDocsWithField) throws IOException {
+ public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException {
if (parser == null) {
throw new NullPointerException();
}
final NumericDocValues valuesIn = reader.getNumericDocValues(field);
if (valuesIn != null) {
- // Not cached here by FieldCacheImpl (cached instead
- // per-thread by SegmentReader):
return valuesIn;
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
@@ -597,28 +636,77 @@ class FieldCacheImpl implements FieldCache {
return DocValues.emptyNumeric();
}
}
- return (NumericDocValues) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField);
+
+ Bits docsWithField = getDocsWithField(reader, field, parser);
+ return ((LongsFromArray) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser))).iterator(docsWithField);
}
}
- static class LongsFromArray extends NumericDocValues implements Accountable {
+ static class LongsFromArray implements Accountable {
private final PackedInts.Reader values;
private final long minValue;
+ private final String field;
- public LongsFromArray(PackedInts.Reader values, long minValue) {
+ public LongsFromArray(String field, PackedInts.Reader values, long minValue) {
+ this.field = field;
this.values = values;
this.minValue = minValue;
}
@Override
- public long get(int docID) {
- return minValue + values.get(docID);
- }
-
- @Override
public long ramBytesUsed() {
return values.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_OBJECT_REF + Long.BYTES;
}
+
+ public NumericDocValues iterator(final Bits docsWithField) {
+ return new NumericDocValues() {
+ int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID >= values.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ if (docsWithField.get(docID)) {
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < values.size()) {
+ docID = target;
+ if (docsWithField.get(docID)) {
+ return docID;
+ } else{
+ return nextDoc();
+ }
+ } else {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return values.size();
+ }
+
+ @Override
+ public long longValue() {
+ return minValue + values.get(docID);
+ }
+ };
+ }
}
static final class LongCache extends Cache {
@@ -627,7 +715,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
- protected Accountable createValue(final LeafReader reader, CacheKey key, boolean setDocsWithField)
+ protected Accountable createValue(final LeafReader reader, CacheKey key)
throws IOException {
final Parser parser = (Parser) key.custom;
@@ -675,16 +763,13 @@ class FieldCacheImpl implements FieldCache {
}
};
- u.uninvert(reader, key.field, setDocsWithField);
-
- if (setDocsWithField) {
- wrapper.setDocsWithField(reader, key.field, u.docsWithField, parser);
- }
+ u.uninvert(reader, key.field);
+ wrapper.setDocsWithField(reader, key.field, u.docsWithField, parser);
GrowableWriterAndMinValue values = valuesRef.get();
if (values == null) {
- return new LongsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0L);
+ return new LongsFromArray(key.field, new PackedInts.NullReader(reader.maxDoc()), 0L);
}
- return new LongsFromArray(values.writer.getMutable(), values.minValue);
+ return new LongsFromArray(key.field, values.writer.getMutable(), values.minValue);
}
}
@@ -704,14 +789,49 @@ class FieldCacheImpl implements FieldCache {
public SortedDocValues iterator() {
final BytesRef term = new BytesRef();
return new SortedDocValues() {
+ private int docID = -1;
@Override
- public int getValueCount() {
- return numOrd;
+ public int docID() {
+ return docID;
}
@Override
- public int getOrd(int docID) {
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID >= docToTermOrd.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ if (docToTermOrd.get(docID) != 0) {
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docToTermOrd.size()) {
+ docID = target;
+ if (docToTermOrd.get(docID) != 0) {
+ return docID;
+ } else{
+ return nextDoc();
+ }
+ } else {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public int ordValue() {
// Subtract 1, matching the 1+ord we did when
// storing, so that missing values, which are 0 in the
// packed ints, are returned as -1 ord:
@@ -719,6 +839,11 @@ class FieldCacheImpl implements FieldCache {
}
@Override
+ public int getValueCount() {
+ return numOrd;
+ }
+
+ @Override
public BytesRef lookupOrd(int ord) {
if (ord < 0) {
throw new IllegalArgumentException("ord must be >=0 (got ord=" + ord + ")");
@@ -769,7 +894,7 @@ class FieldCacheImpl implements FieldCache {
} else if (info.getIndexOptions() == IndexOptions.NONE) {
return DocValues.emptySorted();
}
- SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
+ SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
return impl.iterator();
}
}
@@ -780,7 +905,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
- protected Accountable createValue(LeafReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected Accountable createValue(LeafReader reader, CacheKey key)
throws IOException {
final int maxDoc = reader.maxDoc();
@@ -860,11 +985,54 @@ class FieldCacheImpl implements FieldCache {
this.docToOffset = docToOffset;
}
- public BinaryDocValues iterator() {
- final BytesRef term = new BytesRef();
+ public BinaryDocValues iterator(Bits docsWithField) {
return new BinaryDocValues() {
+
+ final BytesRef term = new BytesRef();
+
+ int docID = -1;
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
@Override
- public BytesRef get(int docID) {
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID >= docToOffset.size()) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ if (docsWithField.get(docID)) {
+ return docID;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target < docToOffset.size()) {
+ docID = target;
+ if (docsWithField.get(docID)) {
+ return docID;
+ } else{
+ return nextDoc();
+ }
+ } else {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
final long pointer = docToOffset.get(docID);
if (pointer == 0) {
term.length = 0;
@@ -892,11 +1060,11 @@ class FieldCacheImpl implements FieldCache {
// TODO: this if DocTermsIndex was already created, we
// should share it...
- public BinaryDocValues getTerms(LeafReader reader, String field, boolean setDocsWithField) throws IOException {
- return getTerms(reader, field, setDocsWithField, PackedInts.FAST);
+ public BinaryDocValues getTerms(LeafReader reader, String field) throws IOException {
+ return getTerms(reader, field, PackedInts.FAST);
}
- public BinaryDocValues getTerms(LeafReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException {
+ public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
if (valuesIn == null) {
valuesIn = reader.getSortedDocValues(field);
@@ -917,8 +1085,8 @@ class FieldCacheImpl implements FieldCache {
return DocValues.emptyBinary();
}
- BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), setDocsWithField);
- return impl.iterator();
+ BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
+ return impl.iterator(getDocsWithField(reader, field, null));
}
static final class BinaryDocValuesCache extends Cache {
@@ -927,7 +1095,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
- protected Accountable createValue(LeafReader reader, CacheKey key, boolean setDocsWithField)
+ protected Accountable createValue(LeafReader reader, CacheKey key)
throws IOException {
// TODO: would be nice to first check if DocTermsIndex
@@ -997,8 +1165,7 @@ class FieldCacheImpl implements FieldCache {
}
final PackedInts.Reader offsetReader = docToOffset.getMutable();
- if (setDocsWithField) {
- wrapper.setDocsWithField(reader, key.field, new Bits() {
+ wrapper.setDocsWithField(reader, key.field, new Bits() {
@Override
public boolean get(int index) {
return offsetReader.get(index) != 0;
@@ -1009,7 +1176,6 @@ class FieldCacheImpl implements FieldCache {
return maxDoc;
}
}, null);
- }
// maybe an int-only impl?
return new BinaryDocValuesImpl(bytes.freeze(true), offsetReader);
}
@@ -1055,7 +1221,7 @@ class FieldCacheImpl implements FieldCache {
}
}
- DocTermOrds dto = (DocTermOrds) caches.get(DocTermOrds.class).get(reader, new CacheKey(field, prefix), false);
+ DocTermOrds dto = (DocTermOrds) caches.get(DocTermOrds.class).get(reader, new CacheKey(field, prefix));
return dto.iterator(reader);
}
@@ -1065,7 +1231,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
- protected Accountable createValue(LeafReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
+ protected Accountable createValue(LeafReader reader, CacheKey key)
throws IOException {
BytesRef prefix = (BytesRef) key.custom;
return new DocTermOrds(reader, null, key.field, prefix);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index 8d49fcb..7825666 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -37,7 +37,6 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.solr.uninverting.FieldCache.CacheEntry;
/**
@@ -274,44 +273,60 @@ public class UninvertingReader extends FilterLeafReader {
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
+ NumericDocValues values = super.getNumericDocValues(field);
+ if (values != null) {
+ return values;
+ }
Type v = getType(field);
if (v != null) {
switch (v) {
- case INTEGER_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.INT_POINT_PARSER, true);
- case FLOAT_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.FLOAT_POINT_PARSER, true);
- case LONG_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LONG_POINT_PARSER, true);
- case DOUBLE_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.DOUBLE_POINT_PARSER, true);
- case LEGACY_INTEGER: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_INT_PARSER, true);
- case LEGACY_FLOAT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_FLOAT_PARSER, true);
- case LEGACY_LONG: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_LONG_PARSER, true);
- case LEGACY_DOUBLE: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_DOUBLE_PARSER, true);
+ case INTEGER_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.INT_POINT_PARSER);
+ case FLOAT_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.FLOAT_POINT_PARSER);
+ case LONG_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LONG_POINT_PARSER);
+ case DOUBLE_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.DOUBLE_POINT_PARSER);
+ case LEGACY_INTEGER: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_INT_PARSER);
+ case LEGACY_FLOAT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_FLOAT_PARSER);
+ case LEGACY_LONG: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_LONG_PARSER);
+ case LEGACY_DOUBLE: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_DOUBLE_PARSER);
}
}
- return super.getNumericDocValues(field);
+ return null;
}
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ BinaryDocValues values = in.getBinaryDocValues(field);
+ if (values != null) {
+ return values;
+ }
Type v = getType(field);
if (v == Type.BINARY) {
- return FieldCache.DEFAULT.getTerms(in, field, true);
+ return FieldCache.DEFAULT.getTerms(in, field);
} else {
- return in.getBinaryDocValues(field);
+ return null;
}
}
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
+ SortedDocValues values = in.getSortedDocValues(field);
+ if (values != null) {
+ return values;
+ }
Type v = getType(field);
if (v == Type.SORTED) {
return FieldCache.DEFAULT.getTermsIndex(in, field);
} else {
- return in.getSortedDocValues(field);
+ return null;
}
}
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+ SortedSetDocValues values = in.getSortedSetDocValues(field);
+ if (values != null) {
+ return values;
+ }
Type v = getType(field);
if (v != null) {
switch (v) {
@@ -325,30 +340,9 @@ public class UninvertingReader extends FilterLeafReader {
return FieldCache.DEFAULT.getDocTermOrds(in, field, null);
}
}
- return in.getSortedSetDocValues(field);
+ return null;
}
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- Type v = getType(field);
- if (v != null) {
- switch (v) {
- case INTEGER_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.INT_POINT_PARSER);
- case FLOAT_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.FLOAT_POINT_PARSER);
- case LONG_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LONG_POINT_PARSER);
- case DOUBLE_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.DOUBLE_POINT_PARSER);
- case LEGACY_INTEGER: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_INT_PARSER);
- case LEGACY_FLOAT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_FLOAT_PARSER);
- case LEGACY_LONG: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_LONG_PARSER);
- case LEGACY_DOUBLE: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_DOUBLE_PARSER);
- default:
- return FieldCache.DEFAULT.getDocsWithField(in, field, null);
- }
- } else {
- return in.getDocsWithField(field);
- }
- }
-
/**
* Returns the field's uninversion type, or null
* if the field doesn't exist or doesn't have a mapping.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
index 90c8b73..d31fc5c 100644
--- a/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
@@ -16,9 +16,11 @@
*/
package org.apache.solr.schema;
-import org.apache.lucene.index.LeafReader;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.core.SolrCore;
@@ -27,8 +29,6 @@ import org.apache.solr.util.RefCounted;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-
public class DocValuesMultiTest extends SolrTestCaseJ4 {
@BeforeClass
@@ -72,13 +72,13 @@ public class DocValuesMultiTest extends SolrTestCaseJ4 {
assertEquals(DocValuesType.SORTED_SET, infos.fieldInfo("intdv").getDocValuesType());
SortedSetDocValues dv = reader.getSortedSetDocValues("stringdv");
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(1, dv.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
dv = reader.getSortedSetDocValues("booldv");
- dv.setDocument(0);
+ assertEquals(0, dv.nextDoc());
assertEquals(0, dv.nextOrd());
assertEquals(1, dv.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
index d59d326..43eefaa 100644
--- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
@@ -16,10 +16,21 @@
*/
package org.apache.solr.schema;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.util.NumericUtils;
import org.apache.solr.SolrTestCaseJ4;
@@ -31,15 +42,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.function.Function;
-import java.util.function.Supplier;
-
public class DocValuesTest extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -84,12 +86,24 @@ public class DocValuesTest extends SolrTestCaseJ4 {
assertEquals(DocValuesType.SORTED, infos.fieldInfo("stringdv").getDocValuesType());
assertEquals(DocValuesType.SORTED, infos.fieldInfo("booldv").getDocValuesType());
- assertEquals((long) Float.floatToIntBits(1), reader.getNumericDocValues("floatdv").get(0));
- assertEquals(2L, reader.getNumericDocValues("intdv").get(0));
- assertEquals(Double.doubleToLongBits(3), reader.getNumericDocValues("doubledv").get(0));
- assertEquals(4L, reader.getNumericDocValues("longdv").get(0));
- assertEquals("solr", reader.getSortedDocValues("stringdv").get(0).utf8ToString());
- assertEquals("T", reader.getSortedDocValues("booldv").get(0).utf8ToString());
+ NumericDocValues dvs = reader.getNumericDocValues("floatdv");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals((long) Float.floatToIntBits(1), dvs.longValue());
+ dvs = reader.getNumericDocValues("intdv");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(2L, dvs.longValue());
+ dvs = reader.getNumericDocValues("doubledv");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(Double.doubleToLongBits(3), dvs.longValue());
+ dvs = reader.getNumericDocValues("longdv");
+ assertEquals(0, dvs.nextDoc());
+ assertEquals(4L, dvs.longValue());
+ SortedDocValues sdv = reader.getSortedDocValues("stringdv");
+ assertEquals(0, sdv.nextDoc());
+ assertEquals("solr", sdv.binaryValue().utf8ToString());
+ sdv = reader.getSortedDocValues("booldv");
+ assertEquals(0, sdv.nextDoc());
+ assertEquals("T", sdv.binaryValue().utf8ToString());
final IndexSchema schema = core.getLatestSchema();
final SchemaField floatDv = schema.getField("floatdv");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/search/TestDocSet.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
index cdddd86..5b44514 100644
--- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
@@ -443,11 +443,6 @@ public class TestDocSet extends LuceneTestCase {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- return null;
- }
-
- @Override
public NumericDocValues getNormValues(String field) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
index e3afed3..b42861a 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
@@ -691,8 +691,18 @@ public class TestRankQueryPlugin extends QParserPlugin {
@Override
public void setScorer(Scorer scorer) throws IOException {}
- public void collect(int doc) {
- list.add(new ScoreDoc(doc+base, (float)values.get(doc)));
+ public void collect(int doc) throws IOException {
+ int valuesDocID = values.docID();
+ if (valuesDocID < doc) {
+ valuesDocID = values.advance(doc);
+ }
+ long value;
+ if (valuesDocID == doc) {
+ value = values.longValue();
+ } else {
+ value = 0;
+ }
+ list.add(new ScoreDoc(doc+base, (float) value));
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java b/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java
index 17b33db..72e2cc6 100644
--- a/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java
+++ b/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.search.function;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SimpleFloatFunction;
@@ -56,7 +58,7 @@ public class NvlValueSourceParser extends ValueSourceParser {
}
@Override
- protected float func(int doc, FunctionValues vals) {
+ protected float func(int doc, FunctionValues vals) throws IOException {
float v = vals.floatVal(doc);
if (v == nvlFloatValue) {
return nvl;
@@ -75,4 +77,4 @@ public class NvlValueSourceParser extends ValueSourceParser {
this.nvlFloatValue = nvlFloatValueArg;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
index 46339a7..67b62cf 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
@@ -115,19 +115,19 @@ public class TestDocTermOrds extends LuceneTestCase {
final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field");
SortedSetDocValues iter = dto.iterator(ar);
- iter.setDocument(0);
+ assertEquals(0, iter.nextDoc());
assertEquals(0, iter.nextOrd());
assertEquals(1, iter.nextOrd());
assertEquals(2, iter.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, iter.nextOrd());
- iter.setDocument(1);
+ assertEquals(1, iter.nextDoc());
assertEquals(3, iter.nextOrd());
assertEquals(4, iter.nextOrd());
assertEquals(5, iter.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, iter.nextOrd());
- iter.setDocument(2);
+ assertEquals(2, iter.nextDoc());
assertEquals(0, iter.nextOrd());
assertEquals(5, iter.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, iter.nextOrd());
@@ -353,7 +353,7 @@ public class TestDocTermOrds extends LuceneTestCase {
TestUtil.nextInt(random(), 2, 10));
- final NumericDocValues docIDToID = FieldCache.DEFAULT.getNumerics(r, "id", FieldCache.LEGACY_INT_PARSER, false);
+ final NumericDocValues docIDToID = FieldCache.DEFAULT.getNumerics(r, "id", FieldCache.LEGACY_INT_PARSER);
/*
for(int docID=0;docID<subR.maxDoc();docID++) {
System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
@@ -405,11 +405,20 @@ public class TestDocTermOrds extends LuceneTestCase {
SortedSetDocValues iter = dto.iterator(r);
for(int docID=0;docID<r.maxDoc();docID++) {
+ assertEquals(docID, docIDToID.nextDoc());
+ if (docID > iter.docID()) {
+ iter.nextDoc();
+ }
+ if (docID < iter.docID()) {
+ int[] answers = idToOrds[(int) docIDToID.longValue()];
+ assertEquals(0, answers.length);
+ continue;
+ }
+
if (VERBOSE) {
- System.out.println("TEST: docID=" + docID + " of " + r.maxDoc() + " (id=" + docIDToID.get(docID) + ")");
+ System.out.println("TEST: docID=" + docID + " of " + r.maxDoc() + " (id=" + docIDToID.longValue() + ")");
}
- iter.setDocument(docID);
- final int[] answers = idToOrds[(int) docIDToID.get(docID)];
+ final int[] answers = idToOrds[(int) docIDToID.longValue()];
int upto = 0;
long ord;
while ((ord = iter.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
@@ -447,7 +456,7 @@ public class TestDocTermOrds extends LuceneTestCase {
SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(getOnlyLeafReader(r1), "foo", null);
assertEquals(3, v.getValueCount());
- v.setDocument(1);
+ assertEquals(1, v.advance(1));
assertEquals(1, v.nextOrd());
iw.close();
@@ -477,12 +486,12 @@ public class TestDocTermOrds extends LuceneTestCase {
SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(ar, "foo", FieldCache.INT32_TERM_PREFIX);
assertEquals(2, v.getValueCount());
-
- v.setDocument(0);
+
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
@@ -519,11 +528,11 @@ public class TestDocTermOrds extends LuceneTestCase {
SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(ar, "foo", FieldCache.INT64_TERM_PREFIX);
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
@@ -654,18 +663,15 @@ public class TestDocTermOrds extends LuceneTestCase {
assertNotNull(DocValues.unwrapSingleton(v)); // actually a single-valued field
assertEquals(2, v.getValueCount());
- v.setDocument(0);
+ assertEquals(0, v.nextDoc());
assertEquals(0, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(1);
+ assertEquals(1, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
- v.setDocument(2);
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- v.setDocument(3);
+ assertEquals(3, v.nextDoc());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
[07/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
index 79149cb..8b9e942 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.mutable.MutableValue;
@@ -33,45 +35,45 @@ public abstract class FloatDocValues extends FunctionValues {
}
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return (byte)floatVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return (short)floatVal(doc);
}
@Override
- public abstract float floatVal(int doc);
+ public abstract float floatVal(int doc) throws IOException;
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return (int)floatVal(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return (long)floatVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return (double)floatVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Float.toString(floatVal(doc));
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? floatVal(doc) : null;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -86,7 +88,7 @@ public abstract class FloatDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = floatVal(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
index aff38ac..2c40fb7 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,45 +37,45 @@ public abstract class IntDocValues extends FunctionValues {
}
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return (byte)intVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return (short)intVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float)intVal(doc);
}
@Override
- public abstract int intVal(int doc);
+ public abstract int intVal(int doc) throws IOException;
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return (long)intVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return (double)intVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Integer.toString(intVal(doc));
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? intVal(doc) : null;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -102,7 +104,7 @@ public abstract class IntDocValues extends FunctionValues {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
int val = intVal(doc);
return val >= ll && val <= uu;
@@ -121,7 +123,7 @@ public abstract class IntDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = intVal(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
index d48bb45..d36afa0 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,50 +37,50 @@ public abstract class LongDocValues extends FunctionValues {
}
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return (byte)longVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return (short)longVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float)longVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return (int)longVal(doc);
}
@Override
- public abstract long longVal(int doc);
+ public abstract long longVal(int doc) throws IOException;
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return (double)longVal(doc);
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return longVal(doc) != 0;
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Long.toString(longVal(doc));
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? longVal(doc) : null;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + '=' + strVal(doc);
}
@@ -111,7 +113,7 @@ public abstract class LongDocValues extends FunctionValues {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
long val = longVal(doc);
return val >= ll && val <= uu;
@@ -130,7 +132,7 @@ public abstract class LongDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = longVal(doc);
mval.exists = exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java
index e6e6b69..169fc95 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.docvalues;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.mutable.MutableValue;
@@ -33,20 +35,20 @@ public abstract class StrDocValues extends FunctionValues {
}
@Override
- public abstract String strVal(int doc);
+ public abstract String strVal(int doc) throws IOException;
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return exists(doc) ? strVal(doc) : null;
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return vs.description() + "='" + strVal(doc) + "'";
}
@@ -61,7 +63,7 @@ public abstract class StrDocValues extends FunctionValues {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.exists = bytesVal(doc, mval.value);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
index 24a65f2..bb33122 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
@@ -19,14 +19,14 @@ package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
-import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
@@ -48,21 +48,42 @@ public class BytesRefFieldSource extends FieldCacheSource {
// TODO: do it cleaner?
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
final BinaryDocValues binaryValues = DocValues.getBinary(readerContext.reader(), field);
- final Bits docsWithField = DocValues.getDocsWithField(readerContext.reader(), field);
return new FunctionValues() {
+ int lastDocID = -1;
+
+ private BytesRef getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = binaryValues.docID();
+ if (doc > curDocID) {
+ curDocID = binaryValues.advance(doc);
+ }
+ if (doc == curDocID) {
+ return binaryValues.binaryValue();
+ } else {
+ return null;
+ }
+ }
@Override
- public boolean exists(int doc) {
- return docsWithField.get(doc);
+ public boolean exists(int doc) throws IOException {
+ return getValueForDoc(doc) != null;
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder target) {
- target.copyBytes(binaryValues.get(doc));
- return target.length() > 0;
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
+ BytesRef value = getValueForDoc(doc);
+ if (value == null || value.length == 0) {
+ return false;
+ } else {
+ target.copyBytes(value);
+ return true;
+ }
}
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
final BytesRefBuilder bytes = new BytesRefBuilder();
return bytesVal(doc, bytes)
? bytes.get().utf8ToString()
@@ -70,12 +91,12 @@ public class BytesRefFieldSource extends FieldCacheSource {
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return strVal(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + '=' + strVal(doc);
}
@@ -90,10 +111,13 @@ public class BytesRefFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
- mval.exists = docsWithField.get(doc);
+ public void fillValue(int doc) throws IOException {
+ BytesRef value = getValueForDoc(doc);
+ mval.exists = value != null;
mval.value.clear();
- mval.value.copyBytes(binaryValues.get(doc));
+ if (value != null) {
+ mval.value.copyBytes(value);
+ }
}
};
}
@@ -108,12 +132,12 @@ public class BytesRefFieldSource extends FieldCacheSource {
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return strVal(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + '=' + strVal(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
index 82d723a..1a936d9 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
@@ -42,7 +42,7 @@ public abstract class ComparisonBoolFunction extends BoolFunction {
}
/** Perform the comparison, returning true or false */
- public abstract boolean compare(int doc, FunctionValues lhs, FunctionValues rhs);
+ public abstract boolean compare(int doc, FunctionValues lhs, FunctionValues rhs) throws IOException;
/** Uniquely identify the operation (ie "gt", "lt" "gte", etc) */
public String name() {
@@ -57,17 +57,17 @@ public abstract class ComparisonBoolFunction extends BoolFunction {
return new BoolDocValues(this) {
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return compare(doc, lhsVal, rhsVal);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return compLabel + "(" + lhsVal.toString(doc) + "," + rhsVal.toString(doc) + ")";
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return lhsVal.exists(doc) && rhsVal.exists(doc);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
index 4880140..f3fd004 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
@@ -48,7 +48,7 @@ public class DefFunction extends MultiFunction {
return new Values(valsArr(sources, fcontext, readerContext)) {
final int upto = valsArr.length - 1;
- private FunctionValues get(int doc) {
+ private FunctionValues get(int doc) throws IOException {
for (int i=0; i<upto; i++) {
FunctionValues vals = valsArr[i];
if (vals.exists(doc)) {
@@ -59,57 +59,57 @@ public class DefFunction extends MultiFunction {
}
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return get(doc).byteVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return get(doc).shortVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return get(doc).floatVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return get(doc).intVal(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return get(doc).longVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return get(doc).doubleVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return get(doc).strVal(doc);
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return get(doc).boolVal(doc);
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder target) {
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
return get(doc).bytesVal(doc, target);
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return get(doc).objectVal(doc);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
// return true if any source is exists?
for (FunctionValues vals : valsArr) {
if (vals.exists(doc)) {
@@ -126,4 +126,4 @@ public class DefFunction extends MultiFunction {
}
};
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java
index c90f94d..e2fd859 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -36,7 +38,7 @@ public class DivFloatFunction extends DualFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return aVals.floatVal(doc) / bVals.floatVal(doc);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
index f66f790..d525dab 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
@@ -19,14 +19,13 @@ package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
-import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueDouble;
@@ -52,17 +51,37 @@ public class DoubleFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
+
+ final NumericDocValues values = DocValues.getNumeric(readerContext.reader(), field);
+
return new DoubleDocValues(this) {
+ int lastDocID;
+
+ private double getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = values.docID();
+ if (doc > curDocID) {
+ curDocID = values.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Double.longBitsToDouble(values.longValue());
+ } else {
+ return 0.0;
+ }
+ }
+
@Override
- public double doubleVal(int doc) {
- return Double.longBitsToDouble(arr.get(doc));
+ public double doubleVal(int doc) throws IOException {
+ return getValueForDoc(doc);
}
@Override
- public boolean exists(int doc) {
- return arr.get(doc) != 0 || valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getValueForDoc(doc);
+ return doc == values.docID();
}
@Override
@@ -76,9 +95,9 @@ public class DoubleFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
- mval.value = doubleVal(doc);
- mval.exists = mval.value != 0 || valid.get(doc);
+ public void fillValue(int doc) throws IOException {
+ mval.value = getValueForDoc(doc);
+ mval.exists = exists(doc);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
index f37ca43..e42ba65 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
@@ -43,7 +43,7 @@ public abstract class DualFloatFunction extends ValueSource {
}
protected abstract String name();
- protected abstract float func(int doc, FunctionValues aVals, FunctionValues bVals);
+ protected abstract float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException;
@Override
public String description() {
@@ -56,7 +56,7 @@ public abstract class DualFloatFunction extends ValueSource {
final FunctionValues bVals = b.getValues(context, readerContext);
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return func(doc, aVals, bVals);
}
/**
@@ -64,11 +64,11 @@ public abstract class DualFloatFunction extends ValueSource {
* <code>exists</code> for the specified doc
*/
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return MultiFunction.allExists(doc, aVals, bVals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
index d0f42a5..2667660 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
@@ -96,25 +95,43 @@ public class EnumFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new IntDocValues(this) {
final MutableValueInt val = new MutableValueInt();
+ int lastDocID;
+
+ private int getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = arr.docID();
+ if (doc > curDocID) {
+ curDocID = arr.advance(doc);
+ }
+ if (doc == curDocID) {
+ return (int) arr.longValue();
+ } else {
+ return 0;
+ }
+ }
+
@Override
- public int intVal(int doc) {
- return (int) arr.get(doc);
+ public int intVal(int doc) throws IOException {
+ return getValueForDoc(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
Integer intValue = intVal(doc);
return intValueToStringValue(intValue);
}
@Override
- public boolean exists(int doc) {
- return valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getValueForDoc(doc);
+ return arr.docID() == doc;
}
@Override
@@ -141,7 +158,7 @@ public class EnumFieldSource extends FieldCacheSource {
return new ValueSourceScorer(readerContext, this) {
@Override
- public boolean matches(int doc) {
+ public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
int val = intVal(doc);
return val >= ll && val <= uu;
@@ -160,9 +177,9 @@ public class EnumFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = intVal(doc);
- mval.exists = valid.get(doc);
+ mval.exists = arr.docID() == doc;
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
index a9a3a85..c4ab4b9 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
@@ -19,14 +19,13 @@ package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
-import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
-import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueFloat;
@@ -52,18 +51,37 @@ public class FloatFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+
return new FloatDocValues(this) {
+ int lastDocID;
+
+ private float getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = arr.docID();
+ if (doc > curDocID) {
+ curDocID = arr.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Float.intBitsToFloat((int)arr.longValue());
+ } else {
+ return 0f;
+ }
+ }
+
@Override
- public float floatVal(int doc) {
- return Float.intBitsToFloat((int)arr.get(doc));
+ public float floatVal(int doc) throws IOException {
+ return getValueForDoc(doc);
}
@Override
- public boolean exists(int doc) {
- return arr.get(doc) != 0 || valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getValueForDoc(doc);
+ return arr.docID() == doc;
}
@Override
@@ -77,9 +95,9 @@ public class FloatFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
+ public void fillValue(int doc) throws IOException {
mval.value = floatVal(doc);
- mval.exists = mval.value != 0 || valid.get(doc);
+ mval.exists = arr.docID() == doc;
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
index 98bd7b4..45d18db 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
@@ -50,57 +50,57 @@ public class IfFunction extends BoolFunction {
return new FunctionValues() {
@Override
- public byte byteVal(int doc) {
+ public byte byteVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.byteVal(doc) : falseVals.byteVal(doc);
}
@Override
- public short shortVal(int doc) {
+ public short shortVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.shortVal(doc) : falseVals.shortVal(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.floatVal(doc) : falseVals.floatVal(doc);
}
@Override
- public int intVal(int doc) {
+ public int intVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.intVal(doc) : falseVals.intVal(doc);
}
@Override
- public long longVal(int doc) {
+ public long longVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.longVal(doc) : falseVals.longVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.doubleVal(doc) : falseVals.doubleVal(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.strVal(doc) : falseVals.strVal(doc);
}
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.boolVal(doc) : falseVals.boolVal(doc);
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder target) {
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
return ifVals.boolVal(doc) ? trueVals.bytesVal(doc, target) : falseVals.bytesVal(doc, target);
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.objectVal(doc) : falseVals.objectVal(doc);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return ifVals.boolVal(doc) ? trueVals.exists(doc) : falseVals.exists(doc);
}
@@ -112,7 +112,7 @@ public class IfFunction extends BoolFunction {
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return "if(" + ifVals.toString(doc) + ',' + trueVals.toString(doc) + ',' + falseVals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
index 686c016..2887e03 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
@@ -26,7 +26,6 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
@@ -52,25 +51,44 @@ public class IntFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+
return new IntDocValues(this) {
final MutableValueInt val = new MutableValueInt();
+ int lastDocID;
+
+ private int getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = arr.docID();
+ if (doc > curDocID) {
+ curDocID = arr.advance(doc);
+ }
+ if (doc == curDocID) {
+ return (int) arr.longValue();
+ } else {
+ return 0;
+ }
+ }
+
@Override
- public int intVal(int doc) {
- return (int) arr.get(doc);
+ public int intVal(int doc) throws IOException {
+ return getValueForDoc(doc);
}
@Override
- public String strVal(int doc) {
+ public String strVal(int doc) throws IOException {
return Integer.toString(intVal(doc));
}
@Override
- public boolean exists(int doc) {
- return arr.get(doc) != 0 || valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getValueForDoc(doc);
+ return arr.docID() == doc;
}
@Override
@@ -84,9 +102,9 @@ public class IntFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
- mval.value = intVal(doc);
- mval.exists = mval.value != 0 || valid.get(doc);
+ public void fillValue(int doc) throws IOException {
+ mval.value = getValueForDoc(doc);
+ mval.exists = arr.docID() == doc;
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
index 9884c7a..db936ba 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
@@ -19,10 +19,10 @@ package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Terms;
@@ -62,27 +62,32 @@ public class JoinDocFreqValueSource extends FieldCacheSource {
return new IntDocValues(this) {
+ int lastDocID = -1;
+
@Override
- public int intVal(int doc)
- {
- try {
- final BytesRef term = terms.get(doc);
+ public int intVal(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = terms.docID();
+ if (doc > curDocID) {
+ curDocID = terms.advance(doc);
+ }
+ if (doc == curDocID) {
+ BytesRef term = terms.binaryValue();
if (termsEnum.seekExact(term)) {
return termsEnum.docFreq();
- } else {
- return 0;
}
- }
- catch (IOException e) {
- throw new RuntimeException("caught exception in function "+description()+" : doc="+doc, e);
}
+ return 0;
}
};
}
@Override
public boolean equals(Object o) {
- if (o.getClass() != JoinDocFreqValueSource.class) return false;
+ if (o.getClass() != JoinDocFreqValueSource.class) return false;
JoinDocFreqValueSource other = (JoinDocFreqValueSource)o;
if( !qfield.equals( other.qfield ) ) return false;
return super.equals(other);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
index 13682af..bbc7419 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
@@ -54,15 +54,15 @@ public class LinearFloatFunction extends ValueSource {
final FunctionValues vals = source.getValues(context, readerContext);
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return vals.floatVal(doc) * slope + intercept;
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return vals.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return slope + "*float(" + vals.toString(doc) + ")+" + intercept;
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
index e231694..2a43f60 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
@@ -26,7 +26,6 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.LongDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueLong;
@@ -64,28 +63,57 @@ public class LongFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
- final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+
return new LongDocValues(this) {
+ int lastDocID;
+
+ private long getValueForDoc(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + doc);
+ }
+ lastDocID = doc;
+ int curDocID = arr.docID();
+ if (doc > curDocID) {
+ curDocID = arr.advance(doc);
+ }
+ if (doc == curDocID) {
+ return arr.longValue();
+ } else {
+ return 0;
+ }
+ }
+
@Override
- public long longVal(int doc) {
- return arr.get(doc);
+ public long longVal(int doc) throws IOException {
+ return getValueForDoc(doc);
}
@Override
- public boolean exists(int doc) {
- return arr.get(doc) != 0 || valid.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getValueForDoc(doc);
+ return arr.docID() == doc;
}
@Override
- public Object objectVal(int doc) {
- return valid.get(doc) ? longToObject(arr.get(doc)) : null;
+ public Object objectVal(int doc) throws IOException {
+ long value = getValueForDoc(doc);
+ if (arr.docID() == doc) {
+ return longToObject(value);
+ } else {
+ return null;
+ }
}
@Override
- public String strVal(int doc) {
- return valid.get(doc) ? longToString(arr.get(doc)) : null;
+ public String strVal(int doc) throws IOException {
+ long value = getValueForDoc(doc);
+ if (arr.docID() == doc) {
+ return longToString(value);
+ } else {
+ return null;
+ }
}
@Override
@@ -104,9 +132,9 @@ public class LongFieldSource extends FieldCacheSource {
}
@Override
- public void fillValue(int doc) {
- mval.value = arr.get(doc);
- mval.exists = mval.value != 0 || valid.get(doc);
+ public void fillValue(int doc) throws IOException {
+ mval.value = getValueForDoc(doc);
+ mval.exists = arr.docID() == doc;
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
index 5f7904b..8abc4fe 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -33,7 +35,7 @@ public class MaxFloatFunction extends MultiFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues[] valsArr) {
+ protected float func(int doc, FunctionValues[] valsArr) throws IOException {
if ( ! exists(doc, valsArr) ) return 0.0f;
float val = Float.NEGATIVE_INFINITY;
@@ -52,7 +54,7 @@ public class MaxFloatFunction extends MultiFloatFunction {
* @see MultiFunction#anyExists
*/
@Override
- protected boolean exists(int doc, FunctionValues[] valsArr) {
+ protected boolean exists(int doc, FunctionValues[] valsArr) throws IOException {
return MultiFunction.anyExists(doc, valsArr);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
index 4190635..ebc1b8b 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -33,7 +35,7 @@ public class MinFloatFunction extends MultiFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues[] valsArr) {
+ protected float func(int doc, FunctionValues[] valsArr) throws IOException {
if ( ! exists(doc, valsArr) ) return 0.0f;
float val = Float.POSITIVE_INFINITY;
@@ -52,7 +54,7 @@ public class MinFloatFunction extends MultiFloatFunction {
* @see MultiFunction#anyExists
*/
@Override
- protected boolean exists(int doc, FunctionValues[] valsArr) {
+ protected boolean exists(int doc, FunctionValues[] valsArr) throws IOException {
return MultiFunction.anyExists(doc, valsArr);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java
index 48d61e1..f368b4a 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java
@@ -39,7 +39,7 @@ public abstract class MultiBoolFunction extends BoolFunction {
protected abstract String name();
- protected abstract boolean func(int doc, FunctionValues[] vals);
+ protected abstract boolean func(int doc, FunctionValues[] vals) throws IOException;
@Override
public BoolDocValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
@@ -51,12 +51,12 @@ public abstract class MultiBoolFunction extends BoolFunction {
return new BoolDocValues(this) {
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return func(doc, vals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder(name());
sb.append('(');
boolean first = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
index 67434b0..f2deb21 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
@@ -38,7 +38,7 @@ public abstract class MultiFloatFunction extends ValueSource {
}
abstract protected String name();
- abstract protected float func(int doc, FunctionValues[] valsArr);
+ abstract protected float func(int doc, FunctionValues[] valsArr) throws IOException;
/**
* Called by {@link FunctionValues#exists} for each document.
*
@@ -48,7 +48,7 @@ public abstract class MultiFloatFunction extends ValueSource {
* @see FunctionValues#exists
* @see MultiFunction#allExists
*/
- protected boolean exists(int doc, FunctionValues[] valsArr) {
+ protected boolean exists(int doc, FunctionValues[] valsArr) throws IOException {
return MultiFunction.allExists(doc, valsArr);
}
@@ -78,14 +78,14 @@ public abstract class MultiFloatFunction extends ValueSource {
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return func(doc, valsArr);
}
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return MultiFloatFunction.this.exists(doc, valsArr);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return MultiFunction.toString(name(), valsArr, doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
index 28182f0..d7db435 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
@@ -48,7 +48,7 @@ public abstract class MultiFunction extends ValueSource {
* @return true if <em>all</em> of the specified <code>values</code>
* {@link FunctionValues#exists} for the specified doc, else false.
*/
- public static boolean allExists(int doc, FunctionValues[] values) {
+ public static boolean allExists(int doc, FunctionValues[] values) throws IOException {
for (FunctionValues v : values) {
if ( ! v.exists(doc) ) {
return false;
@@ -63,7 +63,7 @@ public abstract class MultiFunction extends ValueSource {
* @return true if <em>any</em> of the specified <code>values</code>
* {@link FunctionValues#exists} for the specified doc, else false.
*/
- public static boolean anyExists(int doc, FunctionValues[] values) {
+ public static boolean anyExists(int doc, FunctionValues[] values) throws IOException {
for (FunctionValues v : values) {
if ( v.exists(doc) ) {
return true;
@@ -80,7 +80,7 @@ public abstract class MultiFunction extends ValueSource {
* {@link FunctionValues#exists} for the specified doc, else false.
* @see #anyExists(int,FunctionValues[])
*/
- public static boolean allExists(int doc, FunctionValues values1, FunctionValues values2) {
+ public static boolean allExists(int doc, FunctionValues values1, FunctionValues values2) throws IOException {
return values1.exists(doc) && values2.exists(doc);
}
@@ -92,7 +92,7 @@ public abstract class MultiFunction extends ValueSource {
* {@link FunctionValues#exists} for the specified doc, else false.
* @see #anyExists(int,FunctionValues[])
*/
- public static boolean anyExists(int doc, FunctionValues values1, FunctionValues values2) {
+ public static boolean anyExists(int doc, FunctionValues values1, FunctionValues values2) throws IOException {
return values1.exists(doc) || values2.exists(doc);
}
@@ -129,7 +129,7 @@ public abstract class MultiFunction extends ValueSource {
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return MultiFunction.toString(name(), valsArr, doc);
}
@@ -140,8 +140,7 @@ public abstract class MultiFunction extends ValueSource {
}
}
-
- public static String toString(String name, FunctionValues[] valsArr, int doc) {
+ public static String toString(String name, FunctionValues[] valsArr, int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name).append('(');
boolean firstTime=true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
index 41c0a97..d556454 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+import java.util.Map;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
@@ -24,9 +27,6 @@ import org.apache.lucene.queries.function.docvalues.FloatDocValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
-import java.io.IOException;
-import java.util.Map;
-
/**
* Function that returns {@link TFIDFSimilarity#decodeNormValue(long)}
* for every document.
@@ -68,9 +68,22 @@ public class NormValueSource extends ValueSource {
}
return new FloatDocValues(this) {
+ int lastDocID = -1;
@Override
- public float floatVal(int doc) {
- return similarity.decodeNormValue(norms.get(doc));
+ public float floatVal(int docID) throws IOException {
+ if (docID < lastDocID) {
+ throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " docID=" + docID);
+ }
+ if (docID > norms.docID()) {
+ norms.advance(docID);
+ }
+ long norm;
+ if (docID == norms.docID()) {
+ norm = norms.longValue();
+ } else {
+ norm = 0;
+ }
+ return similarity.decodeNormValue(norm);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java
index 9308b92..bad09e6 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -36,7 +38,7 @@ public class PowFloatFunction extends DualFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues aVals, FunctionValues bVals) {
+ protected float func(int doc, FunctionValues aVals, FunctionValues bVals) throws IOException {
return (float)Math.pow(aVals.floatVal(doc), bVals.floatVal(doc));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java
index 7b5734f..1bb0d4a 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -33,7 +35,7 @@ public class ProductFloatFunction extends MultiFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues[] valsArr) {
+ protected float func(int doc, FunctionValues[] valsArr) throws IOException {
float val = 1.0f;
for (FunctionValues vals : valsArr) {
val *= vals.floatVal(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java
index d902ad6..5492c7f 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java
@@ -64,12 +64,12 @@ public class RangeMapFloatFunction extends ValueSource {
final FunctionValues defaults = (this.defaultVal == null) ? null : defaultVal.getValues(context, readerContext);
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
float val = vals.floatVal(doc);
return (val>=min && val<=max) ? targets.floatVal(doc) : (defaultVal == null ? val : defaults.floatVal(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return "map(" + vals.toString(doc) + ",min=" + min + ",max=" + max + ",target=" + targets.toString(doc) + ",defaultVal=" + (defaults == null ? "null" : (defaults.toString(doc))) + ")";
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
index b96699a..d6b050e 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
@@ -64,15 +64,15 @@ public class ReciprocalFloatFunction extends ValueSource {
final FunctionValues vals = source.getValues(context, readerContext);
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return a/(m*vals.floatVal(doc) + b);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return vals.exists(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return Float.toString(a) + "/("
+ m + "*float(" + vals.toString(doc) + ')'
+ '+' + b + ')';
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
index d810e9c..38bd86a 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
@@ -115,15 +115,15 @@ public class ScaleFloatFunction extends ValueSource {
return new FloatDocValues(this) {
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return vals.exists(doc);
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (vals.floatVal(doc) - minSource) * scale + min;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return "scale(" + vals.toString(doc) + ",toMin=" + min + ",toMax=" + max
+ ",fromMin=" + minSource
+ ",fromMax=" + maxSource
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java
index 4135fb7..d9ad8c4 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java
@@ -41,18 +41,18 @@ public abstract class SimpleBoolFunction extends BoolFunction {
protected abstract String name();
- protected abstract boolean func(int doc, FunctionValues vals);
+ protected abstract boolean func(int doc, FunctionValues vals) throws IOException;
@Override
public BoolDocValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FunctionValues vals = source.getValues(context, readerContext);
return new BoolDocValues(this) {
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
return func(doc, vals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + vals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java
index 364a0a4..f74f8e4 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java
@@ -31,18 +31,18 @@ import java.util.Map;
super(source);
}
- protected abstract float func(int doc, FunctionValues vals);
+ protected abstract float func(int doc, FunctionValues vals) throws IOException;
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FunctionValues vals = source.getValues(context, readerContext);
return new FloatDocValues(this) {
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return func(doc, vals);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + '(' + vals.toString(doc) + ')';
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SortedSetFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SortedSetFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SortedSetFieldSource.java
index 1820d74..d32cc3b 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SortedSetFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SortedSetFieldSource.java
@@ -19,8 +19,8 @@ package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.queries.function.FunctionValues;
@@ -58,14 +58,14 @@ public class SortedSetFieldSource extends FieldCacheSource {
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
SortedSetDocValues sortedSet = DocValues.getSortedSet(readerContext.reader(), field);
SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
- return new DocTermsIndexDocValues(this, view) {
+ return new DocTermsIndexDocValues(this.field, this, view) {
@Override
protected String toTerm(String readableValue) {
return readableValue;
}
@Override
- public Object objectVal(int doc) {
+ public Object objectVal(int doc) throws IOException {
return strVal(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java
index b89a92d..82d542d 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.queries.function.valuesource;
+import java.io.IOException;
+
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -33,11 +35,11 @@ public class SumFloatFunction extends MultiFloatFunction {
}
@Override
- protected float func(int doc, FunctionValues[] valsArr) {
+ protected float func(int doc, FunctionValues[] valsArr) throws IOException {
float val = 0.0f;
for (FunctionValues vals : valsArr) {
val += vals.floatVal(doc);
}
return val;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java
index d5f22fe..768243a 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java
@@ -61,43 +61,42 @@ public class VectorValueSource extends MultiValueSource {
final FunctionValues y = sources.get(1).getValues(context, readerContext);
return new FunctionValues() {
@Override
- public void byteVal(int doc, byte[] vals) {
+ public void byteVal(int doc, byte[] vals) throws IOException {
vals[0] = x.byteVal(doc);
vals[1] = y.byteVal(doc);
}
-
@Override
- public void shortVal(int doc, short[] vals) {
+ public void shortVal(int doc, short[] vals) throws IOException {
vals[0] = x.shortVal(doc);
vals[1] = y.shortVal(doc);
}
@Override
- public void intVal(int doc, int[] vals) {
+ public void intVal(int doc, int[] vals) throws IOException {
vals[0] = x.intVal(doc);
vals[1] = y.intVal(doc);
}
@Override
- public void longVal(int doc, long[] vals) {
+ public void longVal(int doc, long[] vals) throws IOException {
vals[0] = x.longVal(doc);
vals[1] = y.longVal(doc);
}
@Override
- public void floatVal(int doc, float[] vals) {
+ public void floatVal(int doc, float[] vals) throws IOException {
vals[0] = x.floatVal(doc);
vals[1] = y.floatVal(doc);
}
@Override
- public void doubleVal(int doc, double[] vals) {
+ public void doubleVal(int doc, double[] vals) throws IOException {
vals[0] = x.doubleVal(doc);
vals[1] = y.doubleVal(doc);
}
@Override
- public void strVal(int doc, String[] vals) {
+ public void strVal(int doc, String[] vals) throws IOException {
vals[0] = x.strVal(doc);
vals[1] = y.strVal(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return name() + "(" + x.toString(doc) + "," + y.toString(doc) + ")";
}
};
@@ -111,56 +110,56 @@ public class VectorValueSource extends MultiValueSource {
return new FunctionValues() {
@Override
- public void byteVal(int doc, byte[] vals) {
+ public void byteVal(int doc, byte[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].byteVal(doc);
}
}
@Override
- public void shortVal(int doc, short[] vals) {
+ public void shortVal(int doc, short[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].shortVal(doc);
}
}
@Override
- public void floatVal(int doc, float[] vals) {
+ public void floatVal(int doc, float[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].floatVal(doc);
}
}
@Override
- public void intVal(int doc, int[] vals) {
+ public void intVal(int doc, int[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].intVal(doc);
}
}
@Override
- public void longVal(int doc, long[] vals) {
+ public void longVal(int doc, long[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].longVal(doc);
}
}
@Override
- public void doubleVal(int doc, double[] vals) {
+ public void doubleVal(int doc, double[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].doubleVal(doc);
}
}
@Override
- public void strVal(int doc, String[] vals) {
+ public void strVal(int doc, String[] vals) throws IOException {
for (int i = 0; i < valsArr.length; i++) {
vals[i] = valsArr[i].strVal(doc);
}
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
StringBuilder sb = new StringBuilder();
sb.append(name()).append('(');
boolean firstTime = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
index 0caa4ce..540ecf8 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
@@ -160,9 +160,16 @@ public class TestCustomScoreQuery extends FunctionTestSetup {
final NumericDocValues values = DocValues.getNumeric(context.reader(), INT_FIELD);
return new CustomScoreProvider(context) {
@Override
- public float customScore(int doc, float subScore, float valSrcScore) {
+ public float customScore(int doc, float subScore, float valSrcScore) throws IOException {
assertTrue(doc <= context.reader().maxDoc());
- return values.get(doc);
+ if (values.docID() < doc) {
+ values.advance(doc);
+ }
+ if (doc == values.docID()) {
+ return values.longValue();
+ } else {
+ return 0;
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
index 509e0ab..57caac6 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
@@ -392,11 +392,11 @@ public class TestValueSources extends LuceneTestCase {
protected String name() { return "tf_based_exists"; }
@Override
- protected float func(int doc, FunctionValues[] valsArr) {
+ protected float func(int doc, FunctionValues[] valsArr) throws IOException {
return valsArr[0].floatVal(doc);
}
@Override
- protected boolean exists(int doc, FunctionValues[] valsArr) {
+ protected boolean exists(int doc, FunctionValues[] valsArr) throws IOException {
// if tf > 0, then it should exist
return 0 < func(doc, valsArr);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
index 0b1d0c7..40e8dee 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
@@ -18,6 +18,7 @@ package org.apache.lucene.document;
import java.io.IOException;
+import org.apache.lucene.geo.Rectangle;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
@@ -26,7 +27,8 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SloppyMath;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
@@ -65,6 +67,9 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
// the number of times setBottom has been called (adversary protection)
int setBottomCounter = 0;
+ private long[] currentValues = new long[4];
+ private int valuesDocID = -1;
+
public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits) {
this.field = field;
this.latitude = latitude;
@@ -112,19 +117,37 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
public void setTopValue(Double value) {
topValue = value.doubleValue();
}
+
+ private void setValues() throws IOException {
+ if (valuesDocID != currentDocs.docID()) {
+ assert valuesDocID < currentDocs.docID(): " valuesDocID=" + valuesDocID + " vs " + currentDocs.docID();
+ valuesDocID = currentDocs.docID();
+ int count = currentDocs.docValueCount();
+ if (count > currentValues.length) {
+ currentValues = new long[ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_LONG)];
+ }
+ for(int i=0;i<count;i++) {
+ currentValues[i] = currentDocs.nextValue();
+ }
+ }
+ }
@Override
public int compareBottom(int doc) throws IOException {
- currentDocs.setDocument(doc);
-
- int numValues = currentDocs.count();
- if (numValues == 0) {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
+ if (doc < currentDocs.docID()) {
return Double.compare(bottom, Double.POSITIVE_INFINITY);
}
+ setValues();
+
+ int numValues = currentDocs.docValueCount();
+
int cmp = -1;
for (int i = 0; i < numValues; i++) {
- long encoded = currentDocs.valueAt(i);
+ long encoded = currentValues[i];
// test bounding box
int latitudeBits = (int)(encoded >> 32);
@@ -161,6 +184,7 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
LatLonDocValuesField.checkCompatible(info);
}
currentDocs = DocValues.getSortedNumeric(reader, field);
+ valuesDocID = -1;
return this;
}
@@ -176,16 +200,20 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
// TODO: optimize for single-valued case?
// TODO: do all kinds of other optimizations!
- double sortKey(int doc) {
- currentDocs.setDocument(doc);
-
+ double sortKey(int doc) throws IOException {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
double minValue = Double.POSITIVE_INFINITY;
- int numValues = currentDocs.count();
- for (int i = 0; i < numValues; i++) {
- long encoded = currentDocs.valueAt(i);
- double docLatitude = decodeLatitude((int)(encoded >> 32));
- double docLongitude = decodeLongitude((int)(encoded & 0xFFFFFFFF));
- minValue = Math.min(minValue, SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude));
+ if (doc == currentDocs.docID()) {
+ setValues();
+ int numValues = currentDocs.docValueCount();
+ for (int i = 0; i < numValues; i++) {
+ long encoded = currentValues[i];
+ double docLatitude = decodeLatitude((int)(encoded >> 32));
+ double docLongitude = decodeLongitude((int)(encoded & 0xFFFFFFFF));
+ minValue = Math.min(minValue, SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude));
+ }
}
return minValue;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java
index 655e9f3..4cf0a58 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java
@@ -16,6 +16,12 @@
*/
package org.apache.lucene.search;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.DocValues;
@@ -23,12 +29,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.util.Bits;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
-
/**
* Like {@link DocValuesTermsQuery}, but this query only
* runs on a long {@link NumericDocValuesField} or a
@@ -105,14 +105,21 @@ public class DocValuesNumbersQuery extends Query {
@Override
public boolean get(int doc) {
- values.setDocument(doc);
- int count = values.count();
- for(int i=0;i<count;i++) {
- if (numbers.contains(values.valueAt(i))) {
- return true;
+ try {
+ if (doc > values.docID()) {
+ values.advance(doc);
+ }
+ if (doc == values.docID()) {
+ int count = values.docValueCount();
+ for(int i=0;i<count;i++) {
+ if (numbers.contains(values.nextValue())) {
+ return true;
+ }
+ }
}
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
-
return false;
}
[12/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index b9a71a3..894c81a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -19,10 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -325,52 +323,80 @@ class ReadersAndUpdates {
final SegmentWriteState state = new SegmentWriteState(null, trackingDir, info.info, fieldInfos, null, updatesContext, segmentSuffix);
try (final DocValuesConsumer fieldsConsumer = dvFormat.fieldsConsumer(state)) {
// write the numeric updates to a new gen'd docvalues file
- fieldsConsumer.addNumericField(fieldInfo, new Iterable<Number>() {
- final NumericDocValues currentValues = reader.getNumericDocValues(field);
- final Bits docsWithField = reader.getDocsWithField(field);
- final int maxDoc = reader.maxDoc();
- final NumericDocValuesFieldUpdates.Iterator updatesIter = fieldUpdates.iterator();
- @Override
- public Iterator<Number> iterator() {
- updatesIter.reset();
- return new Iterator<Number>() {
-
- int curDoc = -1;
- int updateDoc = updatesIter.nextDoc();
-
- @Override
- public boolean hasNext() {
- return curDoc < maxDoc - 1;
+ fieldsConsumer.addNumericField(fieldInfo, new EmptyDocValuesProducer() {
+ @Override
+ public NumericDocValues getNumeric(FieldInfo fieldInfoIn) throws IOException {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
}
+ final int maxDoc = reader.maxDoc();
- @Override
- public Number next() {
- if (++curDoc >= maxDoc) {
- throw new NoSuchElementException("no more documents to return values for");
+ final NumericDocValuesFieldUpdates.Iterator updatesIter = fieldUpdates.iterator();
+
+ final NumericDocValues currentValues = reader.getNumericDocValues(field);
+ updatesIter.reset();
+
+ // Merge sort of the original doc values with updated doc values:
+ return new NumericDocValues() {
+ // merged docID
+ private int docIDOut = -1;
+
+ // docID from our original doc values
+ private int docIDIn = -1;
+
+ // docID from our updates
+ private int updateDocID = -1;
+
+ private long value;
+
+ @Override
+ public int docID() {
+ return docIDOut;
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ // TODO
+ return 0;
}
- if (curDoc == updateDoc) { // this document has an updated value
- Long value = updatesIter.value(); // either null (unset value) or updated value
- updateDoc = updatesIter.nextDoc(); // prepare for next round
+
+ @Override
+ public long longValue() {
return value;
- } else {
- // no update for this document
- assert curDoc < updateDoc;
- if (currentValues != null && docsWithField.get(curDoc)) {
- // only read the current value if the document had a value before
- return currentValues.get(curDoc);
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docIDIn == docIDOut) {
+ if (currentValues == null) {
+ docIDIn = NO_MORE_DOCS;
+ } else {
+ docIDIn = currentValues.nextDoc();
+ }
+ }
+ if (updateDocID == docIDOut) {
+ updateDocID = updatesIter.nextDoc();
+ }
+ if (docIDIn < updateDocID) {
+ // no update to this doc
+ docIDOut = docIDIn;
+ value = currentValues.longValue();
} else {
- return null;
+ docIDOut = updateDocID;
+ if (docIDOut != NO_MORE_DOCS) {
+ value = updatesIter.value();
+ }
}
+ return docIDOut;
}
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("this iterator does not support removing elements");
- }
- };
- }
- });
+ };
+ }
+ });
}
info.advanceDocValuesGen();
assert !fieldFiles.containsKey(fieldInfo.number);
@@ -398,52 +424,80 @@ class ReadersAndUpdates {
final SegmentWriteState state = new SegmentWriteState(null, trackingDir, info.info, fieldInfos, null, updatesContext, segmentSuffix);
try (final DocValuesConsumer fieldsConsumer = dvFormat.fieldsConsumer(state)) {
// write the binary updates to a new gen'd docvalues file
- fieldsConsumer.addBinaryField(fieldInfo, new Iterable<BytesRef>() {
- final BinaryDocValues currentValues = reader.getBinaryDocValues(field);
- final Bits docsWithField = reader.getDocsWithField(field);
- final int maxDoc = reader.maxDoc();
- final BinaryDocValuesFieldUpdates.Iterator updatesIter = fieldUpdates.iterator();
- @Override
- public Iterator<BytesRef> iterator() {
- updatesIter.reset();
- return new Iterator<BytesRef>() {
-
- int curDoc = -1;
- int updateDoc = updatesIter.nextDoc();
-
- @Override
- public boolean hasNext() {
- return curDoc < maxDoc - 1;
+
+ fieldsConsumer.addBinaryField(fieldInfo, new EmptyDocValuesProducer() {
+ @Override
+ public BinaryDocValues getBinary(FieldInfo fieldInfoIn) throws IOException {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
}
-
- @Override
- public BytesRef next() {
- if (++curDoc >= maxDoc) {
- throw new NoSuchElementException("no more documents to return values for");
+ final int maxDoc = reader.maxDoc();
+
+ final BinaryDocValuesFieldUpdates.Iterator updatesIter = fieldUpdates.iterator();
+ updatesIter.reset();
+
+ final BinaryDocValues currentValues = reader.getBinaryDocValues(field);
+
+ // Merge sort of the original doc values with updated doc values:
+ return new BinaryDocValues() {
+ // merged docID
+ private int docIDOut = -1;
+
+ // docID from our original doc values
+ private int docIDIn = -1;
+
+ // docID from our updates
+ private int updateDocID = -1;
+
+ private BytesRef value;
+
+ @Override
+ public int docID() {
+ return docIDOut;
}
- if (curDoc == updateDoc) { // this document has an updated value
- BytesRef value = updatesIter.value(); // either null (unset value) or updated value
- updateDoc = updatesIter.nextDoc(); // prepare for next round
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ return currentValues.cost();
+ }
+
+ @Override
+ public BytesRef binaryValue() {
return value;
- } else {
- // no update for this document
- assert curDoc < updateDoc;
- if (currentValues != null && docsWithField.get(curDoc)) {
- // only read the current value if the document had a value before
- return currentValues.get(curDoc);
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docIDIn == docIDOut) {
+ if (currentValues == null) {
+ docIDIn = NO_MORE_DOCS;
+ } else {
+ docIDIn = currentValues.nextDoc();
+ }
+ }
+ if (updateDocID == docIDOut) {
+ updateDocID = updatesIter.nextDoc();
+ }
+ if (docIDIn < updateDocID) {
+ // no update to this doc
+ docIDOut = docIDIn;
+ value = currentValues.binaryValue();
} else {
- return null;
+ docIDOut = updateDocID;
+ if (docIDOut != NO_MORE_DOCS) {
+ value = updatesIter.value();
+ }
}
+ return docIDOut;
}
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("this iterator does not support removing elements");
- }
- };
- }
- });
+ };
+ }
+ });
}
info.advanceDocValuesGen();
assert !fieldFiles.containsKey(fieldInfo.number);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
index e3662d5..1ccfd39 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
@@ -31,7 +31,6 @@ import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.RamUsageEstimator;
/** Encapsulates multiple producers when there are docvalues updates as one producer */
@@ -129,13 +128,6 @@ class SegmentDocValuesProducer extends DocValuesProducer {
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.name);
- assert dvProducer != null;
- return dvProducer.getDocsWithField(field);
- }
-
- @Override
public void checkIntegrity() throws IOException {
for (DocValuesProducer producer : dvProducers) {
producer.checkIntegrity();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
index ed0a06e..ebbca4f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
@@ -191,14 +191,10 @@ public final class SegmentReader extends CodecReader {
try {
core.decRef();
} finally {
- try {
- super.doClose();
- } finally {
- if (docValuesProducer instanceof SegmentDocValuesProducer) {
- segDocValues.decRef(((SegmentDocValuesProducer)docValuesProducer).dvGens);
- } else if (docValuesProducer != null) {
- segDocValues.decRef(Collections.singletonList(-1L));
- }
+ if (docValuesProducer instanceof SegmentDocValuesProducer) {
+ segDocValues.decRef(((SegmentDocValuesProducer)docValuesProducer).dvGens);
+ } else if (docValuesProducer != null) {
+ segDocValues.decRef(Collections.singletonList(-1L));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SingletonSortedNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedNumericDocValues.java
index 69f3714..d95f0c0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedNumericDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedNumericDocValues.java
@@ -16,9 +16,8 @@
*/
package org.apache.lucene.index;
+import java.io.IOException;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.Bits.MatchAllBits;
/**
* Exposes multi-valued view over a single-valued instance.
@@ -28,42 +27,58 @@ import org.apache.lucene.util.Bits.MatchAllBits;
*/
final class SingletonSortedNumericDocValues extends SortedNumericDocValues {
private final NumericDocValues in;
- private final Bits docsWithField;
private long value;
- private int count;
- public SingletonSortedNumericDocValues(NumericDocValues in, Bits docsWithField) {
+ public SingletonSortedNumericDocValues(NumericDocValues in) {
+ if (in.docID() != -1) {
+ throw new IllegalStateException("iterator has already been used: docID=" + in.docID());
+ }
this.in = in;
- this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField;
}
/** Return the wrapped {@link NumericDocValues} */
public NumericDocValues getNumericDocValues() {
+ if (in.docID() != -1) {
+ throw new IllegalStateException("iterator has already been used: docID=" + in.docID());
+ }
return in;
}
-
- /** Return the wrapped {@link Bits} */
- public Bits getDocsWithField() {
- return docsWithField;
+
+ @Override
+ public int docID() {
+ return in.docID();
}
@Override
- public void setDocument(int doc) {
- value = in.get(doc);
- if (docsWithField != null && value == 0 && docsWithField.get(doc) == false) {
- count = 0;
- } else {
- count = 1;
+ public int nextDoc() throws IOException {
+ int docID = in.nextDoc();
+ if (docID != NO_MORE_DOCS) {
+ value = in.longValue();
}
+ return docID;
}
-
+
+ @Override
+ public int advance(int target) throws IOException {
+ int docID = in.advance(target);
+ if (docID != NO_MORE_DOCS) {
+ value = in.longValue();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
@Override
- public long valueAt(int index) {
+ public long nextValue() {
return value;
}
@Override
- public int count() {
- return count;
+ public int docValueCount() {
+ return 1;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
index 5077cd8..88f8f32 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
@@ -17,30 +17,43 @@
package org.apache.lucene.index;
+import java.io.IOException;
+
import org.apache.lucene.util.BytesRef;
/**
- * Exposes multi-valued view over a single-valued instance.
+ * Exposes multi-valued iterator view over a single-valued iterator.
* <p>
* This can be used if you want to have one multi-valued implementation
* that works for single or multi-valued types.
*/
-final class SingletonSortedSetDocValues extends RandomAccessOrds {
+final class SingletonSortedSetDocValues extends SortedSetDocValues {
private final SortedDocValues in;
private long currentOrd;
private long ord;
/** Creates a multi-valued view over the provided SortedDocValues */
public SingletonSortedSetDocValues(SortedDocValues in) {
+ if (in.docID() != -1) {
+ throw new IllegalStateException("iterator has already been used: docID=" + in.docID());
+ }
this.in = in;
}
/** Return the wrapped {@link SortedDocValues} */
public SortedDocValues getSortedDocValues() {
+ if (in.docID() != -1) {
+ throw new IllegalStateException("iterator has already been used: docID=" + in.docID());
+ }
return in;
}
@Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
public long nextOrd() {
long v = currentOrd;
currentOrd = NO_MORE_ORDS;
@@ -48,8 +61,21 @@ final class SingletonSortedSetDocValues extends RandomAccessOrds {
}
@Override
- public void setDocument(int docID) {
- currentOrd = ord = in.getOrd(docID);
+ public int nextDoc() throws IOException {
+ int docID = in.nextDoc();
+ if (docID != NO_MORE_DOCS) {
+ currentOrd = ord = in.ordValue();
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ int docID = in.advance(target);
+ if (docID != NO_MORE_DOCS) {
+ currentOrd = ord = in.ordValue();
+ }
+ return docID;
}
@Override
@@ -69,17 +95,12 @@ final class SingletonSortedSetDocValues extends RandomAccessOrds {
}
@Override
- public long ordAt(int index) {
- return ord;
- }
-
- @Override
- public int cardinality() {
- return (int) (ord >>> 63) ^ 1;
+ public TermsEnum termsEnum() {
+ return in.termsEnum();
}
@Override
- public TermsEnum termsEnum() {
- return in.termsEnum();
+ public long cost() {
+ return in.cost();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
index 2742247..618d375 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
@@ -224,7 +224,7 @@ public final class SlowCodecReaderWrapper {
return new DocValuesProducer() {
@Override
- public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
return reader.getNumericDocValues(field.name);
}
@@ -249,11 +249,6 @@ public final class SlowCodecReaderWrapper {
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- return reader.getDocsWithField(field.name);
- }
-
- @Override
public void checkIntegrity() throws IOException {
// We already checkIntegrity the entire reader up front
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
index e9a55a3..0f43f6a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
@@ -20,13 +20,16 @@ package org.apache.lucene.index;
import org.apache.lucene.util.BytesRef;
/**
- * A per-document byte[] with presorted values.
+ * A per-document byte[] with presorted values. This is fundamentally an
+ * iterator over the int ord values per document, with random access APIs
+ * to resolve an int ord to BytesRef.
* <p>
* Per-Document values in a SortedDocValues are deduplicated, dereferenced,
* and sorted into a dictionary of unique values. A pointer to the
* dictionary value (ordinal) can be retrieved for each document. Ordinals
* are dense and in increasing sorted order.
*/
+
public abstract class SortedDocValues extends BinaryDocValues {
/** Sole constructor. (For invocation by subclass
@@ -34,35 +37,27 @@ public abstract class SortedDocValues extends BinaryDocValues {
protected SortedDocValues() {}
/**
- * Returns the ordinal for the specified docID.
- * @param docID document ID to lookup
+ * Returns the ordinal for the current docID.
* @return ordinal for the document: this is dense, starts at 0, then
* increments by 1 for the next value in sorted order. Note that
* missing values are indicated by -1.
*/
- public abstract int getOrd(int docID);
+ public abstract int ordValue();
/** Retrieves the value for the specified ordinal. The returned
* {@link BytesRef} may be re-used across calls to {@link #lookupOrd(int)}
* so make sure to {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want
* to keep it around.
* @param ord ordinal to lookup (must be >= 0 and < {@link #getValueCount()})
- * @see #getOrd(int)
+ * @see #ordValue()
*/
public abstract BytesRef lookupOrd(int ord);
- /**
- * Returns the number of unique values.
- * @return number of unique values in this SortedDocValues. This is
- * also equivalent to one plus the maximum ordinal.
- */
- public abstract int getValueCount();
-
private final BytesRef empty = new BytesRef();
@Override
- public BytesRef get(int docID) {
- int ord = getOrd(docID);
+ public BytesRef binaryValue() {
+ int ord = ordValue();
if (ord == -1) {
return empty;
} else {
@@ -70,6 +65,13 @@ public abstract class SortedDocValues extends BinaryDocValues {
}
}
+ /**
+ * Returns the number of unique values.
+ * @return number of unique values in this SortedDocValues. This is
+ * also equivalent to one plus the maximum ordinal.
+ */
+ public abstract int getValueCount();
+
/** If {@code key} exists, returns its ordinal, else
* returns {@code -insertionPoint-1}, like {@code
* Arrays.binarySearch}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
index 6517218..f199fcc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
@@ -19,8 +19,6 @@ package org.apache.lucene.index;
import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.ByteBlockPool;
@@ -41,6 +39,7 @@ class SortedDocValuesWriter extends DocValuesWriter {
private final FieldInfo fieldInfo;
private static final int EMPTY_ORD = -1;
+ private int nonEmptyCount;
public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
this.fieldInfo = fieldInfo;
@@ -72,6 +71,7 @@ class SortedDocValuesWriter extends DocValuesWriter {
}
addOneValue(value);
+ nonEmptyCount++;
}
@Override
@@ -120,91 +120,87 @@ class SortedDocValuesWriter extends DocValuesWriter {
}
dvConsumer.addSortedField(fieldInfo,
-
- // ord -> value
- new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new ValuesIterator(sortedValues, valueCount, hash);
- }
- },
-
- // doc -> ord
- new Iterable<Number>() {
+ new EmptyDocValuesProducer() {
@Override
- public Iterator<Number> iterator() {
- return new OrdsIterator(ordMap, maxDoc, ords);
+ public SortedDocValues getSorted(FieldInfo fieldInfoIn) {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedSortedDocValues(hash, valueCount, maxDoc, ords, sortedValues, ordMap, nonEmptyCount);
}
});
}
- // iterates over the unique values we have in ram
- private static class ValuesIterator implements Iterator<BytesRef> {
- final int sortedValues[];
+ private static class BufferedSortedDocValues extends SortedDocValues {
final BytesRefHash hash;
+ final int maxDoc;
final BytesRef scratch = new BytesRef();
+ final int[] sortedValues;
+ final int[] ordMap;
+ final int cost;
final int valueCount;
- int ordUpto;
-
- ValuesIterator(int sortedValues[], int valueCount, BytesRefHash hash) {
- this.sortedValues = sortedValues;
- this.valueCount = valueCount;
+ private int docID = -1;
+ private int ord;
+ final PackedLongValues.Iterator iter;
+
+ public BufferedSortedDocValues(BytesRefHash hash, int valueCount, int maxDoc, PackedLongValues docToOrd, int[] sortedValues, int[] ordMap, int cost) {
this.hash = hash;
+ this.valueCount = valueCount;
+ this.maxDoc = maxDoc;
+ this.sortedValues = sortedValues;
+ this.iter = docToOrd.iterator();
+ this.ordMap = ordMap;
+ this.cost = cost;
}
@Override
- public boolean hasNext() {
- return ordUpto < valueCount;
+ public int docID() {
+ return docID;
}
@Override
- public BytesRef next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID >= maxDoc) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ ord = (int) iter.next();
+ if (ord != -1) {
+ ord = ordMap[ord];
+ break;
+ }
}
- hash.get(sortedValues[ordUpto], scratch);
- ordUpto++;
- return scratch;
+ return docID;
}
@Override
- public void remove() {
+ public int advance(int target) {
throw new UnsupportedOperationException();
}
- }
-
- // iterates over the ords for each doc we have in ram
- private static class OrdsIterator implements Iterator<Number> {
- final PackedLongValues.Iterator iter;
- final int ordMap[];
- final int maxDoc;
- int docUpto;
-
- OrdsIterator(int ordMap[], int maxDoc, PackedLongValues ords) {
- this.ordMap = ordMap;
- this.maxDoc = maxDoc;
- assert ords.size() == maxDoc;
- this.iter = ords.iterator();
+
+ @Override
+ public long cost() {
+ return cost;
}
-
+
@Override
- public boolean hasNext() {
- return docUpto < maxDoc;
+ public int ordValue() {
+ return ord;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- int ord = (int) iter.next();
- docUpto++;
- return ord == -1 ? ord : ordMap[ord];
+ public BytesRef lookupOrd(int ord) {
+ assert ord >= 0 && ord < sortedValues.length;
+ assert sortedValues[ord] >= 0 && sortedValues[ord] < sortedValues.length;
+ hash.get(sortedValues[ord], scratch);
+ return scratch;
}
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public int getValueCount() {
+ return valueCount;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValues.java
index 8b85a6c..0a888f8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValues.java
@@ -16,31 +16,30 @@
*/
package org.apache.lucene.index;
+import java.io.IOException;
+
+import org.apache.lucene.search.DocIdSetIterator;
+
/**
* A list of per-document numeric values, sorted
* according to {@link Long#compare(long, long)}.
*/
-public abstract class SortedNumericDocValues {
+public abstract class SortedNumericDocValues extends DocIdSetIterator {
/** Sole constructor. (For invocation by subclass
- * constructors, typically implicit.) */
+ * constructors, typically implicit.) */
protected SortedNumericDocValues() {}
/**
- * Positions to the specified document
- */
- public abstract void setDocument(int doc);
-
- /**
* Retrieve the value for the current document at the specified index.
* An index ranges from {@code 0} to {@code count()-1}.
*/
- public abstract long valueAt(int index);
+ public abstract long nextValue() throws IOException;
/**
- * Retrieves the count of values for the current document.
- * This may be zero if a document has no values.
+ * Retrieves the number of values for the current document. This must always
+ * be greater than zero.
*/
- public abstract int count();
+ public abstract int docValueCount();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
index 5ce9a91..7db5d98 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
@@ -19,8 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
@@ -109,73 +107,82 @@ class SortedNumericDocValuesWriter extends DocValuesWriter {
final PackedLongValues valueCounts = pendingCounts.build();
dvConsumer.addSortedNumericField(fieldInfo,
- // doc -> valueCount
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new CountIterator(valueCounts);
- }
- },
-
- // values
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new ValuesIterator(values);
- }
- });
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo fieldInfoIn) {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedSortedNumericDocValues(values, valueCounts);
+ }
+ });
}
-
- // iterates over the values for each doc we have in ram
- private static class ValuesIterator implements Iterator<Number> {
- final PackedLongValues.Iterator iter;
- ValuesIterator(PackedLongValues values) {
- iter = values.iterator();
+ private static class BufferedSortedNumericDocValues extends SortedNumericDocValues {
+ final PackedLongValues.Iterator valuesIter;
+ final PackedLongValues.Iterator valueCountsIter;
+ final int maxDoc;
+ final long cost;
+ private int docID = -1;
+ private int valueCount;
+ private int valueUpto;
+
+ public BufferedSortedNumericDocValues(PackedLongValues values, PackedLongValues valueCounts) {
+ valuesIter = values.iterator();
+ valueCountsIter = valueCounts.iterator();
+ maxDoc = Math.toIntExact(valueCounts.size());
+ cost = values.size();
}
@Override
- public boolean hasNext() {
- return iter.hasNext();
+ public int docID() {
+ return docID;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
+ public int nextDoc() {
+
+ // consume any un-consumed values from current doc
+ while(valueUpto < valueCount) {
+ valuesIter.next();
+ valueUpto++;
+ }
+
+ while (true) {
+ docID++;
+ if (docID == maxDoc) {
+ docID = NO_MORE_DOCS;
+ break;
+ } else {
+ valueCount = Math.toIntExact(valueCountsIter.next());
+ if (valueCount > 0) {
+ valueUpto = 0;
+ break;
+ }
+ }
}
- return iter.next();
+ return docID;
}
@Override
- public void remove() {
+ public int advance(int target) {
throw new UnsupportedOperationException();
}
- }
-
- private static class CountIterator implements Iterator<Number> {
- final PackedLongValues.Iterator iter;
-
- CountIterator(PackedLongValues valueCounts) {
- this.iter = valueCounts.iterator();
- }
@Override
- public boolean hasNext() {
- return iter.hasNext();
+ public int docValueCount() {
+ return valueCount;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- return iter.next();
+ public long nextValue() {
+ valueUpto++;
+ return valuesIter.next();
}
-
+
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public long cost() {
+ return cost;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
index f68efcc..c442c5f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
@@ -17,17 +17,20 @@
package org.apache.lucene.index;
+import java.io.IOException;
+
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
/**
- * A per-document set of presorted byte[] values.
+ * A multi-valued version of {@link SortedDocValues}.
* <p>
- * Per-Document values in a SortedDocValues are deduplicated, dereferenced,
+ * Per-Document values in a SortedSetDocValues are deduplicated, dereferenced,
* and sorted into a dictionary of unique values. A pointer to the
* dictionary value (ordinal) can be retrieved for each document. Ordinals
* are dense and in increasing sorted order.
*/
-public abstract class SortedSetDocValues {
+public abstract class SortedSetDocValues extends DocIdSetIterator {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
@@ -39,20 +42,15 @@ public abstract class SortedSetDocValues {
public static final long NO_MORE_ORDS = -1;
/**
- * Returns the next ordinal for the current document (previously
- * set by {@link #setDocument(int)}.
+ * Returns the next ordinal for the current document.
* @return next ordinal for the document, or {@link #NO_MORE_ORDS}.
* ordinals are dense, start at 0, then increment by 1 for
* the next value in sorted order.
*/
- public abstract long nextOrd();
-
- /**
- * Sets iteration to the specified docID
- * @param docID document ID
- */
- public abstract void setDocument(int docID);
+ public abstract long nextOrd() throws IOException;
+ // TODO: should we have a docValueCount, like SortedNumeric?
+
/** Retrieves the value for the specified ordinal. The returned
* {@link BytesRef} may be re-used across calls to lookupOrd so make sure to
* {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
index 3f3beb3..b474d86 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
@@ -20,8 +20,6 @@ import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
@@ -44,8 +42,8 @@ class SortedSetDocValuesWriter extends DocValuesWriter {
private final FieldInfo fieldInfo;
private int currentDoc;
private int currentValues[] = new int[8];
- private int currentUpto = 0;
- private int maxCount = 0;
+ private int currentUpto;
+ private int maxCount;
public SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
this.fieldInfo = fieldInfo;
@@ -128,9 +126,7 @@ class SortedSetDocValuesWriter extends DocValuesWriter {
if (currentUpto == currentValues.length) {
currentValues = ArrayUtil.grow(currentValues, currentValues.length+1);
- // reserve additional space for max # values per-doc
- // when flushing, we need an int[] to sort the mapped-ords within the doc
- iwBytesUsed.addAndGet((currentValues.length - currentUpto) * 2 * Integer.BYTES);
+ iwBytesUsed.addAndGet((currentValues.length - currentUpto) * Integer.BYTES);
}
currentValues[currentUpto] = termID;
@@ -158,150 +154,105 @@ class SortedSetDocValuesWriter extends DocValuesWriter {
for(int ord=0;ord<valueCount;ord++) {
ordMap[sortedValues[ord]] = ord;
}
-
dvConsumer.addSortedSetField(fieldInfo,
-
- // ord -> value
- new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new ValuesIterator(sortedValues, valueCount, hash);
- }
- },
-
- // doc -> ordCount
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new OrdCountIterator(maxDoc, ordCounts);
- }
- },
-
- // ords
- new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new OrdsIterator(ordMap, maxCountPerDoc, ords, ordCounts);
- }
- });
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedSetDocValues getSortedSet(FieldInfo fieldInfoIn) {
+ if (fieldInfoIn != fieldInfo) {
+ throw new IllegalArgumentException("wrong fieldInfo");
+ }
+ return new BufferedSortedSetDocValues(sortedValues, ordMap, hash, ords, ordCounts, maxCount);
+ }
+ });
}
- // iterates over the unique values we have in ram
- private static class ValuesIterator implements Iterator<BytesRef> {
- final int sortedValues[];
+ private static class BufferedSortedSetDocValues extends SortedSetDocValues {
+ private int docID = -1;
+ final int[] sortedValues;
+ final int[] ordMap;
final BytesRefHash hash;
+ final PackedLongValues ords;
+ final PackedLongValues ordCounts;
final BytesRef scratch = new BytesRef();
- final int valueCount;
- int ordUpto;
+ final PackedLongValues.Iterator ordsIter;
+ final PackedLongValues.Iterator ordCountsIter;
+ final int currentDoc[];
- ValuesIterator(int sortedValues[], int valueCount, BytesRefHash hash) {
+ private int ordCount;
+ private int ordUpto;
+
+ public BufferedSortedSetDocValues(int[] sortedValues, int[] ordMap, BytesRefHash hash, PackedLongValues ords, PackedLongValues ordCounts, int maxCount) {
+ this.currentDoc = new int[maxCount];
this.sortedValues = sortedValues;
- this.valueCount = valueCount;
+ this.ordMap = ordMap;
this.hash = hash;
+ this.ords = ords;
+ this.ordCounts = ordCounts;
+ this.ordsIter = ords.iterator();
+ this.ordCountsIter = ordCounts.iterator();
}
@Override
- public boolean hasNext() {
- return ordUpto < valueCount;
+ public int docID() {
+ return docID;
}
@Override
- public BytesRef next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
+ public int nextDoc() {
+ // consume any un-consumed ords from current doc
+ while (ordUpto < ordCount) {
+ ordsIter.next();
+ ordUpto++;
+ }
+ while (true) {
+ docID++;
+ if (docID == ordCounts.size()) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ ordCount = (int) ordCountsIter.next();
+ if (ordCount > 0) {
+ for(int i=0;i<ordCount;i++) {
+ currentDoc[i] = ordMap[Math.toIntExact(ordsIter.next())];
+ }
+ Arrays.sort(currentDoc, 0, ordCount);
+ ordUpto = 0;
+ break;
+ }
}
- hash.get(sortedValues[ordUpto], scratch);
- ordUpto++;
- return scratch;
- }
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
- // iterates over the ords for each doc we have in ram
- private static class OrdsIterator implements Iterator<Number> {
- final PackedLongValues.Iterator iter;
- final PackedLongValues.Iterator counts;
- final int ordMap[];
- final long numOrds;
- long ordUpto;
-
- final int currentDoc[];
- int currentUpto;
- int currentLength;
-
- OrdsIterator(int ordMap[], int maxCount, PackedLongValues ords, PackedLongValues ordCounts) {
- this.currentDoc = new int[maxCount];
- this.ordMap = ordMap;
- this.numOrds = ords.size();
- this.iter = ords.iterator();
- this.counts = ordCounts.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return ordUpto < numOrds;
+ return docID;
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- while (currentUpto == currentLength) {
- // refill next doc, and sort remapped ords within the doc.
- currentUpto = 0;
- currentLength = (int) counts.next();
- for (int i = 0; i < currentLength; i++) {
- currentDoc[i] = ordMap[(int) iter.next()];
- }
- Arrays.sort(currentDoc, 0, currentLength);
+ public long nextOrd() {
+ if (ordUpto == ordCount) {
+ return NO_MORE_ORDS;
+ } else {
+ return currentDoc[ordUpto++];
}
- int ord = currentDoc[currentUpto];
- currentUpto++;
- ordUpto++;
- // TODO: make reusable Number
- return ord;
}
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public long cost() {
+ return ordCounts.size();
}
- }
-
- private static class OrdCountIterator implements Iterator<Number> {
- final PackedLongValues.Iterator iter;
- final int maxDoc;
- int docUpto;
-
- OrdCountIterator(int maxDoc, PackedLongValues ordCounts) {
- this.maxDoc = maxDoc;
- assert ordCounts.size() == maxDoc;
- this.iter = ordCounts.iterator();
- }
-
+
@Override
- public boolean hasNext() {
- return docUpto < maxDoc;
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
}
@Override
- public Number next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- docUpto++;
- // TODO: make reusable Number
- return iter.next();
+ public long getValueCount() {
+ return ordMap.length;
}
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public BytesRef lookupOrd(long ord) {
+ assert ord >= 0 && ord < ordMap.length: "ord=" + ord + " is out of bounds 0 .. " + (ordMap.length-1);
+ hash.get(sortedValues[Math.toIntExact(ord)], scratch);
+ return scratch;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/Sorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Sorter.java b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
index cf75c18..75816ba 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Sorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
@@ -17,10 +17,11 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Comparator;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
@@ -28,6 +29,8 @@ import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Sorts documents of a given index by returning a permutation on the document
* IDs.
@@ -85,7 +88,6 @@ final class Sorter {
/** Compare docID1 against docID2. The contract for the return value is the
* same as {@link Comparator#compare(Object, Object)}. */
public abstract int compare(int docID1, int docID2);
-
}
private static final class DocValueSorter extends TimSorter {
@@ -198,6 +200,178 @@ final class Sorter {
};
}
+ /** We cannot use the {@link FieldComparator} API because that API requires that you send it docIDs in order. Note that this API
+ * allocates arrays[maxDoc] to hold the native values needed for comparison, but 1) they are transient (only alive while sorting this one
+ * segment), and 2) in the typical index sorting case, they are only used to sort newly flushed segments, which will be smaller than
+ * merged segments. */
+ private static DocComparator getDocComparator(LeafReader reader, SortField sortField) throws IOException {
+
+ int maxDoc = reader.maxDoc();
+
+ switch(sortField.getType()) {
+
+ case STRING:
+ {
+ final int missingOrd;
+ if (sortField.getMissingValue() == SortField.STRING_LAST) {
+ missingOrd = Integer.MAX_VALUE;
+ } else {
+ missingOrd = Integer.MIN_VALUE;
+ }
+
+ final int[] ords = new int[reader.maxDoc()];
+ Arrays.fill(ords, missingOrd);
+ SortedDocValues sorted = DocValues.getSorted(reader, sortField.getField());
+ int docID;
+ while ((docID = sorted.nextDoc()) != NO_MORE_DOCS) {
+ ords[docID] = sorted.ordValue();
+ }
+
+ final int reverseMul;
+ if (sortField.getReverse()) {
+ reverseMul = -1;
+ } else {
+ reverseMul = 1;
+ }
+
+ return new DocComparator() {
+ @Override
+ public int compare(int docID1, int docID2) {
+ return reverseMul * Integer.compare(ords[docID1], ords[docID2]);
+ }
+ };
+ }
+
+ case LONG:
+ {
+ long[] values = new long[maxDoc];
+ NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
+
+ if (sortField.getMissingValue() != null) {
+ Arrays.fill(values, (Long) sortField.getMissingValue());
+ }
+ while (true) {
+ int docID = dvs.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ values[docID] = dvs.longValue();
+ }
+
+ final int reverseMul;
+ if (sortField.getReverse()) {
+ reverseMul = -1;
+ } else {
+ reverseMul = 1;
+ }
+
+ return new DocComparator() {
+ @Override
+ public int compare(int docID1, int docID2) {
+ return reverseMul * Long.compare(values[docID1], values[docID2]);
+ }
+ };
+ }
+
+ case INT:
+ {
+ int[] values = new int[maxDoc];
+ NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
+
+ if (sortField.getMissingValue() != null) {
+ Arrays.fill(values, (Integer) sortField.getMissingValue());
+ }
+
+ while (true) {
+ int docID = dvs.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ values[docID] = (int) dvs.longValue();
+ }
+
+ final int reverseMul;
+ if (sortField.getReverse()) {
+ reverseMul = -1;
+ } else {
+ reverseMul = 1;
+ }
+
+ return new DocComparator() {
+ @Override
+ public int compare(int docID1, int docID2) {
+ return reverseMul * Integer.compare(values[docID1], values[docID2]);
+ }
+ };
+ }
+
+ case DOUBLE:
+ {
+ double[] values = new double[maxDoc];
+ NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
+
+ if (sortField.getMissingValue() != null) {
+ Arrays.fill(values, (Double) sortField.getMissingValue());
+ }
+ while (true) {
+ int docID = dvs.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ values[docID] = Double.longBitsToDouble(dvs.longValue());
+ }
+
+ final int reverseMul;
+ if (sortField.getReverse()) {
+ reverseMul = -1;
+ } else {
+ reverseMul = 1;
+ }
+
+ return new DocComparator() {
+ @Override
+ public int compare(int docID1, int docID2) {
+ return reverseMul * Double.compare(values[docID1], values[docID2]);
+ }
+ };
+ }
+
+ case FLOAT:
+ {
+ float[] values = new float[maxDoc];
+ NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
+
+ if (sortField.getMissingValue() != null) {
+ Arrays.fill(values, (Float) sortField.getMissingValue());
+ }
+ while (true) {
+ int docID = dvs.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ values[docID] = Float.intBitsToFloat((int) dvs.longValue());
+ }
+
+ final int reverseMul;
+ if (sortField.getReverse()) {
+ reverseMul = -1;
+ } else {
+ reverseMul = 1;
+ }
+
+ return new DocComparator() {
+ @Override
+ public int compare(int docID1, int docID2) {
+ return reverseMul * Float.compare(values[docID1], values[docID2]);
+ }
+ };
+ }
+
+ default:
+ throw new IllegalArgumentException("unhandled SortField.getType()=" + sortField.getType());
+ }
+ }
+
/**
* Returns a mapping from the old document ID to its new location in the
* sorted index. Implementations can use the auxiliary
@@ -213,33 +387,25 @@ final class Sorter {
DocMap sort(LeafReader reader) throws IOException {
SortField fields[] = sort.getSort();
final int reverseMul[] = new int[fields.length];
- final LeafFieldComparator comparators[] = new LeafFieldComparator[fields.length];
+ final DocComparator comparators[] = new DocComparator[fields.length];
for (int i = 0; i < fields.length; i++) {
- reverseMul[i] = fields[i].getReverse() ? -1 : 1;
- comparators[i] = fields[i].getComparator(1, i).getLeafComparator(reader.getContext());
- comparators[i].setScorer(FAKESCORER);
+ comparators[i] = getDocComparator(reader, fields[i]);
}
+
final DocComparator comparator = new DocComparator() {
@Override
public int compare(int docID1, int docID2) {
- try {
- for (int i = 0; i < comparators.length; i++) {
- // TODO: would be better if copy() didnt cause a term lookup in TermOrdVal & co,
- // the segments are always the same here...
- comparators[i].copy(0, docID1);
- comparators[i].setBottom(0);
- int comp = reverseMul[i] * comparators[i].compareBottom(docID2);
- if (comp != 0) {
- return comp;
- }
+ for (int i = 0; i < comparators.length; i++) {
+ int comp = comparators[i].compare(docID1, docID2);
+ if (comp != 0) {
+ return comp;
}
- return Integer.compare(docID1, docID2); // docid order tiebreak
- } catch (IOException e) {
- throw new RuntimeException(e);
}
+ return Integer.compare(docID1, docID2); // docid order tiebreak
}
};
+
return sort(reader.maxDoc(), comparator);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
index c1476d0..3e34436 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
@@ -19,8 +19,9 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
-import org.apache.lucene.index.Sorter.DocMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.store.IndexInput;
@@ -29,11 +30,16 @@ import org.apache.lucene.store.RAMFile;
import org.apache.lucene.store.RAMInputStream;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* An {@link org.apache.lucene.index.LeafReader} which supports sorting documents by a given
* {@link Sort}. This is package private and is only used by Lucene when it needs to merge
@@ -150,59 +156,112 @@ class SortingLeafReader extends FilterLeafReader {
private static class SortingBinaryDocValues extends BinaryDocValues {
- private final BinaryDocValues in;
- private final Sorter.DocMap docMap;
+ private final CachedBinaryDVs dvs;
+ private int docID = -1;
- SortingBinaryDocValues(BinaryDocValues in, Sorter.DocMap docMap) {
- this.in = in;
- this.docMap = docMap;
+ public SortingBinaryDocValues(CachedBinaryDVs dvs) {
+ this.dvs = dvs;
}
@Override
- public BytesRef get(int docID) {
- return in.get(docMap.newToOld(docID));
+ public int nextDoc() {
+ if (docID+1 == dvs.docsWithField.length()) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = dvs.docsWithField.nextSetBit(docID+1);
+ }
+
+ return docID;
}
- }
- private static class SortingNumericDocValues extends NumericDocValues {
+ @Override
+ public int docID() {
+ return docID;
+ }
- private final NumericDocValues in;
- private final Sorter.DocMap docMap;
+ @Override
+ public int advance(int target) {
+ docID = dvs.docsWithField.nextSetBit(target);
+ return docID;
+ }
- public SortingNumericDocValues(final NumericDocValues in, Sorter.DocMap docMap) {
- this.in = in;
- this.docMap = docMap;
+ @Override
+ public BytesRef binaryValue() {
+ return dvs.values[docID];
}
@Override
- public long get(int docID) {
- return in.get(docMap.newToOld(docID));
+ public long cost() {
+ return dvs.docsWithField.cardinality();
}
}
- private static class SortingSortedNumericDocValues extends SortedNumericDocValues {
+ private final Map<String,CachedNumericDVs> cachedNumericDVs = new HashMap<>();
- private final SortedNumericDocValues in;
- private final Sorter.DocMap docMap;
+ private static class CachedNumericDVs {
+ private final long[] values;
+ private final BitSet docsWithField;
- SortingSortedNumericDocValues(SortedNumericDocValues in, DocMap docMap) {
- this.in = in;
- this.docMap = docMap;
+ public CachedNumericDVs(long[] values, BitSet docsWithField) {
+ this.values = values;
+ this.docsWithField = docsWithField;
+ }
+ }
+
+ private final Map<String,CachedBinaryDVs> cachedBinaryDVs = new HashMap<>();
+
+ private static class CachedBinaryDVs {
+ // TODO: at least cutover to BytesRefArray here:
+ private final BytesRef[] values;
+ private final BitSet docsWithField;
+
+ public CachedBinaryDVs(BytesRef[] values, BitSet docsWithField) {
+ this.values = values;
+ this.docsWithField = docsWithField;
+ }
+ }
+
+ private final Map<String,int[]> cachedSortedDVs = new HashMap<>();
+
+ private static class SortingNumericDocValues extends NumericDocValues {
+
+ private final CachedNumericDVs dvs;
+ private int docID = -1;
+
+ public SortingNumericDocValues(CachedNumericDVs dvs) {
+ this.dvs = dvs;
}
@Override
- public int count() {
- return in.count();
+ public int docID() {
+ return docID;
}
@Override
- public void setDocument(int doc) {
- in.setDocument(docMap.newToOld(doc));
+ public int nextDoc() {
+ if (docID+1 == dvs.docsWithField.length()) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = dvs.docsWithField.nextSetBit(docID+1);
+ }
+
+ return docID;
}
@Override
- public long valueAt(int index) {
- return in.valueAt(index);
+ public int advance(int target) {
+ docID = dvs.docsWithField.nextSetBit(target);
+ return docID;
+ }
+
+ @Override
+ public long longValue() {
+ return dvs.values[docID];
+ }
+
+ @Override
+ public long cost() {
+ return dvs.docsWithField.cardinality();
}
}
@@ -292,58 +351,135 @@ class SortingLeafReader extends FilterLeafReader {
private static class SortingSortedDocValues extends SortedDocValues {
private final SortedDocValues in;
- private final Sorter.DocMap docMap;
+ private final int[] ords;
+ private int docID = -1;
- SortingSortedDocValues(SortedDocValues in, Sorter.DocMap docMap) {
+ SortingSortedDocValues(SortedDocValues in, int[] ords) {
this.in = in;
- this.docMap = docMap;
+ this.ords = ords;
+ assert ords != null;
}
@Override
- public int getOrd(int docID) {
- return in.getOrd(docMap.newToOld(docID));
+ public int docID() {
+ return docID;
}
@Override
- public BytesRef lookupOrd(int ord) {
- return in.lookupOrd(ord);
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == ords.length) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ if (ords[docID] != -1) {
+ break;
+ }
+ // skip missing docs
+ }
+
+ return docID;
}
@Override
- public int getValueCount() {
- return in.getValueCount();
+ public int advance(int target) {
+ if (target >= ords.length) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = target;
+ if (ords[docID] == -1) {
+ nextDoc();
+ }
+ }
+ return docID;
+ }
+
+ @Override
+ public int ordValue() {
+ return ords[docID];
}
@Override
- public BytesRef get(int docID) {
- return in.get(docMap.newToOld(docID));
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ return in.lookupOrd(ord);
}
@Override
- public int lookupTerm(BytesRef key) {
- return in.lookupTerm(key);
+ public int getValueCount() {
+ return in.getValueCount();
}
}
+ // TODO: pack long[][] into an int[] (offset) and long[] instead:
+ private final Map<String,long[][]> cachedSortedSetDVs = new HashMap<>();
+
private static class SortingSortedSetDocValues extends SortedSetDocValues {
private final SortedSetDocValues in;
- private final Sorter.DocMap docMap;
+ private final long[][] ords;
+ private int docID = -1;
+ private int ordUpto;
- SortingSortedSetDocValues(SortedSetDocValues in, Sorter.DocMap docMap) {
+ SortingSortedSetDocValues(SortedSetDocValues in, long[][] ords) {
this.in = in;
- this.docMap = docMap;
+ this.ords = ords;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == ords.length) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ if (ords[docID] != null) {
+ break;
+ }
+ // skip missing docs
+ }
+ ordUpto = 0;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= ords.length) {
+ docID = NO_MORE_DOCS;
+ } else {
+ docID = target;
+ if (ords[docID] == null) {
+ nextDoc();
+ } else {
+ ordUpto = 0;
+ }
+ }
+ return docID;
}
@Override
public long nextOrd() {
- return in.nextOrd();
+ if (ordUpto == ords[docID].length) {
+ return NO_MORE_ORDS;
+ } else {
+ return ords[docID][ordUpto++];
+ }
}
@Override
- public void setDocument(int docID) {
- //System.out.println(" slr.sssdv.setDocument docID=" + docID + " this=" + this);
- in.setDocument(docMap.newToOld(docID));
+ public long cost() {
+ return in.cost();
}
@Override
@@ -355,10 +491,71 @@ class SortingLeafReader extends FilterLeafReader {
public long getValueCount() {
return in.getValueCount();
}
+ }
+
+ private final Map<String,long[][]> cachedSortedNumericDVs = new HashMap<>();
+
+ private static class SortingSortedNumericDocValues extends SortedNumericDocValues {
+ private final SortedNumericDocValues in;
+ private final long[][] values;
+ private int docID = -1;
+ private int upto;
+
+ SortingSortedNumericDocValues(SortedNumericDocValues in, long[][] values) {
+ this.in = in;
+ this.values = values;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ while (true) {
+ docID++;
+ if (docID == values.length) {
+ docID = NO_MORE_DOCS;
+ break;
+ }
+ if (values[docID] != null) {
+ break;
+ }
+ // skip missing docs
+ }
+ upto = 0;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= values.length) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ } else {
+ docID = target-1;
+ return nextDoc();
+ }
+ }
+
+ @Override
+ public long nextValue() {
+ if (upto == values[docID].length) {
+ throw new AssertionError();
+ } else {
+ return values[docID][upto++];
+ }
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
@Override
- public long lookupTerm(BytesRef key) {
- return in.lookupTerm(key);
+ public int docValueCount() {
+ return values[docID].length;
}
}
@@ -816,13 +1013,30 @@ class SortingLeafReader extends FilterLeafReader {
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
- BinaryDocValues oldDocValues = in.getBinaryDocValues(field);
- if (oldDocValues == null) {
- return null;
- } else {
- return new SortingBinaryDocValues(oldDocValues, docMap);
+ final BinaryDocValues oldDocValues = in.getBinaryDocValues(field);
+ if (oldDocValues == null) return null;
+ CachedBinaryDVs dvs;
+ synchronized (cachedBinaryDVs) {
+ dvs = cachedBinaryDVs.get(field);
+ if (dvs == null) {
+ FixedBitSet docsWithField = new FixedBitSet(maxDoc());
+ BytesRef[] values = new BytesRef[maxDoc()];
+ while (true) {
+ int docID = oldDocValues.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ int newDocID = docMap.oldToNew(docID);
+ docsWithField.set(newDocID);
+ values[newDocID] = BytesRef.deepCopyOf(oldDocValues.binaryValue());
+ }
+ dvs = new CachedBinaryDVs(values, docsWithField);
+ cachedBinaryDVs.put(field, dvs);
+ }
}
+ return new SortingBinaryDocValues(dvs);
}
+
@Override
public Bits getLiveDocs() {
@@ -844,21 +1058,58 @@ class SortingLeafReader extends FilterLeafReader {
}
}
+ private final Map<String,CachedNumericDVs> cachedNorms = new HashMap<>();
+
@Override
public NumericDocValues getNormValues(String field) throws IOException {
- final NumericDocValues norm = in.getNormValues(field);
- if (norm == null) {
- return null;
- } else {
- return new SortingNumericDocValues(norm, docMap);
+ final NumericDocValues oldNorms = in.getNormValues(field);
+ if (oldNorms == null) return null;
+ CachedNumericDVs norms;
+ synchronized (cachedNorms) {
+ norms = cachedNorms.get(field);
+ if (norms == null) {
+ FixedBitSet docsWithField = new FixedBitSet(maxDoc());
+ long[] values = new long[maxDoc()];
+ while (true) {
+ int docID = oldNorms.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ int newDocID = docMap.oldToNew(docID);
+ docsWithField.set(newDocID);
+ values[newDocID] = oldNorms.longValue();
+ }
+ norms = new CachedNumericDVs(values, docsWithField);
+ cachedNorms.put(field, norms);
+ }
}
+ return new SortingNumericDocValues(norms);
}
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
final NumericDocValues oldDocValues = in.getNumericDocValues(field);
if (oldDocValues == null) return null;
- return new SortingNumericDocValues(oldDocValues, docMap);
+ CachedNumericDVs dvs;
+ synchronized (cachedNumericDVs) {
+ dvs = cachedNumericDVs.get(field);
+ if (dvs == null) {
+ FixedBitSet docsWithField = new FixedBitSet(maxDoc());
+ long[] values = new long[maxDoc()];
+ while (true) {
+ int docID = oldDocValues.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ int newDocID = docMap.oldToNew(docID);
+ docsWithField.set(newDocID);
+ values[newDocID] = oldDocValues.longValue();
+ }
+ dvs = new CachedNumericDVs(values, docsWithField);
+ cachedNumericDVs.put(field, dvs);
+ }
+ }
+ return new SortingNumericDocValues(dvs);
}
@Override
@@ -867,39 +1118,88 @@ class SortingLeafReader extends FilterLeafReader {
final SortedNumericDocValues oldDocValues = in.getSortedNumericDocValues(field);
if (oldDocValues == null) {
return null;
- } else {
- return new SortingSortedNumericDocValues(oldDocValues, docMap);
}
+
+ long[][] values;
+ synchronized (cachedSortedNumericDVs) {
+ values = cachedSortedNumericDVs.get(field);
+ if (values == null) {
+ values = new long[maxDoc()][];
+ int docID;
+ while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {
+ int newDocID = docMap.oldToNew(docID);
+ long[] docValues = new long[oldDocValues.docValueCount()];
+ for(int i=0;i<docValues.length;i++) {
+ docValues[i] = oldDocValues.nextValue();
+ }
+ values[newDocID] = docValues;
+ }
+ cachedSortedNumericDVs.put(field, values);
+ }
+ }
+
+ return new SortingSortedNumericDocValues(oldDocValues, values);
}
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
- SortedDocValues sortedDV = in.getSortedDocValues(field);
- if (sortedDV == null) {
+ SortedDocValues oldDocValues = in.getSortedDocValues(field);
+ if (oldDocValues == null) {
return null;
- } else {
- return new SortingSortedDocValues(sortedDV, docMap);
}
+
+ int[] ords;
+ synchronized (cachedSortedDVs) {
+ ords = cachedSortedDVs.get(field);
+ if (ords == null) {
+ ords = new int[maxDoc()];
+ Arrays.fill(ords, -1);
+ int docID;
+ while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {
+ int newDocID = docMap.oldToNew(docID);
+ ords[newDocID] = oldDocValues.ordValue();
+ }
+ cachedSortedDVs.put(field, ords);
+ }
+ }
+
+ return new SortingSortedDocValues(oldDocValues, ords);
}
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
- SortedSetDocValues sortedSetDV = in.getSortedSetDocValues(field);
- if (sortedSetDV == null) {
+ SortedSetDocValues oldDocValues = in.getSortedSetDocValues(field);
+ if (oldDocValues == null) {
return null;
- } else {
- return new SortingSortedSetDocValues(sortedSetDV, docMap);
}
- }
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- Bits bits = in.getDocsWithField(field);
- if (bits == null || bits instanceof Bits.MatchAllBits || bits instanceof Bits.MatchNoBits) {
- return bits;
- } else {
- return new SortingBits(bits, docMap);
+ long[][] ords;
+ synchronized (cachedSortedSetDVs) {
+ ords = cachedSortedSetDVs.get(field);
+ if (ords == null) {
+ ords = new long[maxDoc()][];
+ int docID;
+ while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {
+ int newDocID = docMap.oldToNew(docID);
+ long[] docOrds = new long[1];
+ int upto = 0;
+ while (true) {
+ long ord = oldDocValues.nextOrd();
+ if (ord == NO_MORE_ORDS) {
+ break;
+ }
+ if (upto == docOrds.length) {
+ docOrds = ArrayUtil.grow(docOrds);
+ }
+ docOrds[upto++] = ord;
+ }
+ ords[newDocID] = Arrays.copyOfRange(docOrds, 0, upto);
+ }
+ cachedSortedSetDVs.put(field, ords);
+ }
}
+
+ return new SortingSortedSetDocValues(oldDocValues, ords);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java
index 43c3c05..0bf7a8e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java
@@ -145,13 +145,21 @@ public final class DocValuesRewriteMethod extends MultiTermQuery.RewriteMethod {
@Override
public boolean get(int doc) {
- fcsi.setDocument(doc);
- for (long ord = fcsi.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = fcsi.nextOrd()) {
- if (termSet.get(ord)) {
- return true;
+ try {
+ if (doc > fcsi.docID()) {
+ fcsi.advance(doc);
}
+ if (doc == fcsi.docID()) {
+ for (long ord = fcsi.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = fcsi.nextOrd()) {
+ if (termSet.get(ord)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
- return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
index f5ccc7c..002375d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
@@ -101,7 +101,7 @@ final class ExactPhraseScorer extends Scorer {
}
@Override
- public float score() {
+ public float score() throws IOException {
return docScorer.score(docID(), freq);
}
[06/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java
index 44f3f8c..129b9348 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java
@@ -183,13 +183,21 @@ public final class DocValuesRangeQuery extends Query {
@Override
public boolean get(int doc) {
- values.setDocument(doc);
- final int count = values.count();
- for (int i = 0; i < count; ++i) {
- final long value = values.valueAt(i);
- if (value >= min && value <= max) {
- return true;
+ try {
+ if (doc > values.docID()) {
+ values.advance(doc);
}
+ if (doc == values.docID()) {
+ final int count = values.docValueCount();
+ for (int i = 0; i < count; ++i) {
+ final long value = values.nextValue();
+ if (value >= min && value <= max) {
+ return true;
+ }
+ }
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
return false;
}
@@ -241,11 +249,19 @@ public final class DocValuesRangeQuery extends Query {
@Override
public boolean get(int doc) {
- values.setDocument(doc);
- for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
- if (ord >= minOrd && ord <= maxOrd) {
- return true;
+ try {
+ if (doc > values.docID()) {
+ values.advance(doc);
+ }
+ if (doc == values.docID()) {
+ for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+ if (ord >= minOrd && ord <= maxOrd) {
+ return true;
+ }
+ }
}
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java
index 4be4b18..92037a8 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java
@@ -165,11 +165,19 @@ public class DocValuesTermsQuery extends Query {
@Override
public boolean get(int doc) {
- values.setDocument(doc);
- for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
- if (bits.get(ord)) {
- return true;
+ try {
+ if (doc > values.docID()) {
+ values.advance(doc);
}
+ if (doc == values.docID()) {
+ for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+ if (bits.get(ord)) {
+ return true;
+ }
+ }
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
index 9c5cf7f..0a1755c 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java
@@ -360,7 +360,7 @@ class TermAutomatonScorer extends Scorer {
}
@Override
- public float score() {
+ public float score() throws IOException {
// TODO: we could probably do better here, e.g. look @ freqs of actual terms involved in this doc and score differently
return docScorer.score(docID, freq);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
index 9d29330..ff61ff6 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
@@ -203,7 +203,6 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase {
int dimensions = ranges[0][0].numDimensions();
int iters = atLeast(25);
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
@@ -250,8 +249,10 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase {
public boolean needsScores() { return false; }
});
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for (int docID=0; docID<maxDoc; ++docID) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
index 1d8b4b0..b1424e0 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
@@ -67,22 +67,23 @@ public abstract class BBoxSimilarityValueSource extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
//? limit to Rect or call getBoundingBox()? latter would encourage bad practice
final Rectangle rect = (Rectangle) shapeValues.objectVal(doc);
return rect==null ? 0 : score(rect, null);
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return shapeValues.exists(doc);
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
final Rectangle rect = (Rectangle) shapeValues.objectVal(doc);
- if (rect == null)
+ if (rect == null) {
return Explanation.noMatch("no rect");
+ }
AtomicReference<Explanation> explanation = new AtomicReference<>();
score(rect, explanation);
return explanation.get();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
index 2bfbfd9..ef9f54e 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
@@ -16,18 +16,17 @@
*/
package org.apache.lucene.spatial.bbox;
-import org.locationtech.spatial4j.shape.Rectangle;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.util.Map;
+import org.locationtech.spatial4j.shape.Rectangle;
/**
* A ValueSource in which the indexed Rectangle is returned from
@@ -56,41 +55,62 @@ class BBoxValueSource extends ValueSource {
final NumericDocValues maxX = DocValues.getNumeric(reader, strategy.field_maxX);
final NumericDocValues maxY = DocValues.getNumeric(reader, strategy.field_maxY);
- final Bits validBits = DocValues.getDocsWithField(reader, strategy.field_minX);//could have chosen any field
//reused
final Rectangle rect = strategy.getSpatialContext().makeRectangle(0,0,0,0);
return new FunctionValues() {
+ private int lastDocID = -1;
+
+ private double getDocValue(NumericDocValues values, int doc) throws IOException {
+ int curDocID = values.docID();
+ if (doc > curDocID) {
+ curDocID = values.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Double.longBitsToDouble(values.longValue());
+ } else {
+ return 0.0;
+ }
+ }
+
@Override
- public Object objectVal(int doc) {
- if (!validBits.get(doc)) {
+ public Object objectVal(int doc) throws IOException {
+ if (doc < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + doc);
+ }
+ lastDocID = doc;
+
+ double minXValue = getDocValue(minX, doc);
+ if (minX.docID() != doc) {
return null;
} else {
- rect.reset(
- Double.longBitsToDouble(minX.get(doc)), Double.longBitsToDouble(maxX.get(doc)),
- Double.longBitsToDouble(minY.get(doc)), Double.longBitsToDouble(maxY.get(doc)));
+ double minYValue = getDocValue(minY, doc);
+ double maxXValue = getDocValue(maxX, doc);
+ double maxYValue = getDocValue(maxY, doc);
+ rect.reset(minXValue, maxXValue, minYValue, maxYValue);
return rect;
}
}
@Override
- public String strVal(int doc) {//TODO support WKT output once Spatial4j does
+ public String strVal(int doc) throws IOException {//TODO support WKT output once Spatial4j does
Object v = objectVal(doc);
return v == null ? null : v.toString();
}
@Override
- public boolean exists(int doc) {
- return validBits.get(doc);
+ public boolean exists(int doc) throws IOException {
+ getDocValue(minX, doc);
+ return minX.docID() == doc;
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
return Explanation.match(Float.NaN, toString(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + '=' + strVal(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
index c2093f2..d9c45f1 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
@@ -24,10 +24,6 @@ import java.io.FilterOutputStream;
import java.io.IOException;
import java.util.Map;
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.io.BinaryCodec;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.BinaryDocValues;
@@ -47,6 +43,10 @@ import org.apache.lucene.spatial.util.ShapePredicateValueSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.io.BinaryCodec;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
/**
@@ -143,7 +143,11 @@ public class SerializedDVStrategy extends SpatialStrategy {
return new Bits() {
@Override
public boolean get(int index) {
- return predFuncValues.boolVal(index);
+ try {
+ return predFuncValues.boolVal(index);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
}
@Override
@@ -196,21 +200,28 @@ public class SerializedDVStrategy extends SpatialStrategy {
int bytesRefDoc = -1;
BytesRefBuilder bytesRef = new BytesRefBuilder();
- boolean fillBytes(int doc) {
+ boolean fillBytes(int doc) throws IOException {
if (bytesRefDoc != doc) {
- bytesRef.copyBytes(docValues.get(doc));
+ if (docValues.docID() < doc) {
+ docValues.advance(doc);
+ }
+ if (docValues.docID() == doc) {
+ bytesRef.copyBytes(docValues.binaryValue());
+ } else {
+ bytesRef.clear();
+ }
bytesRefDoc = doc;
}
return bytesRef.length() != 0;
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return fillBytes(doc);
}
@Override
- public boolean bytesVal(int doc, BytesRefBuilder target) {
+ public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
target.clear();
if (fillBytes(doc)) {
target.copyBytes(bytesRef);
@@ -221,7 +232,7 @@ public class SerializedDVStrategy extends SpatialStrategy {
}
@Override
- public Object objectVal(int docId) {
+ public Object objectVal(int docId) throws IOException {
if (!fillBytes(docId))
return null;
DataInputStream dataInput = new DataInputStream(
@@ -234,12 +245,12 @@ public class SerializedDVStrategy extends SpatialStrategy {
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
return Explanation.match(Float.NaN, toString(doc));
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + "=" + objectVal(doc);//TODO truncate?
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
index 73d25ca..d43d4e8 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
@@ -52,7 +52,7 @@ public class CachingDoubleValueSource extends ValueSource {
return new FunctionValues() {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
Integer key = Integer.valueOf( base+doc );
Double v = cache.get( key );
if( v == null ) {
@@ -63,12 +63,12 @@ public class CachingDoubleValueSource extends ValueSource {
}
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float)doubleVal(doc);
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return doubleVal(doc)+"";
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
index 7be2433..6993adb 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
@@ -76,7 +76,7 @@ public class DistanceToShapeValueSource extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
Shape shape = (Shape) shapeValues.objectVal(doc);
if (shape == null || shape.isEmpty())
return nullValue;
@@ -85,7 +85,7 @@ public class DistanceToShapeValueSource extends ValueSource {
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
Explanation exp = super.explain(doc);
List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
details.add(shapeValues.explain(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
index 257dc67..9c2c0e3 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
@@ -69,7 +69,7 @@ public class ShapeAreaValueSource extends ValueSource {
return new DoubleDocValues(this) {
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
Shape shape = (Shape) shapeValues.objectVal(doc);
if (shape == null || shape.isEmpty())
return 0;//or NaN?
@@ -79,12 +79,12 @@ public class ShapeAreaValueSource extends ValueSource {
}
@Override
- public boolean exists(int doc) {
+ public boolean exists(int doc) throws IOException {
return shapeValues.exists(doc);
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
Explanation exp = super.explain(doc);
List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
details.add(shapeValues.explain(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
index 08c1e43..d0dc51f 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
@@ -72,7 +72,7 @@ public class ShapePredicateValueSource extends ValueSource {
return new BoolDocValues(this) {
@Override
- public boolean boolVal(int doc) {
+ public boolean boolVal(int doc) throws IOException {
Shape indexedShape = (Shape) shapeValues.objectVal(doc);
if (indexedShape == null)
return false;
@@ -80,7 +80,7 @@ public class ShapePredicateValueSource extends ValueSource {
}
@Override
- public Explanation explain(int doc) {
+ public Explanation explain(int doc) throws IOException {
Explanation exp = super.explain(doc);
List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
details.add(shapeValues.explain(doc));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
index 7cab3fe..e1f8f7c 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
@@ -16,15 +16,14 @@
*/
package org.apache.lucene.spatial.vector;
+import org.apache.lucene.index.NumericDocValues;
import org.locationtech.spatial4j.distance.DistanceCalculator;
import org.locationtech.spatial4j.shape.Point;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.util.Bits;
import java.io.IOException;
import java.util.Map;
@@ -67,33 +66,47 @@ public class DistanceValueSource extends ValueSource {
final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());
- final Bits validX = DocValues.getDocsWithField(reader, strategy.getFieldNameX());
- final Bits validY = DocValues.getDocsWithField(reader, strategy.getFieldNameY());
return new FunctionValues() {
+ private int lastDocID = -1;
+
private final Point from = DistanceValueSource.this.from;
private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();
private final double nullValue =
(strategy.getSpatialContext().isGeo() ? 180 * multiplier : Double.MAX_VALUE);
+ private double getDocValue(NumericDocValues values, int doc) throws IOException {
+ int curDocID = values.docID();
+ if (doc > curDocID) {
+ curDocID = values.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Double.longBitsToDouble(values.longValue());
+ } else {
+ return 0.0;
+ }
+ }
+
@Override
- public float floatVal(int doc) {
+ public float floatVal(int doc) throws IOException {
return (float) doubleVal(doc);
}
@Override
- public double doubleVal(int doc) {
+ public double doubleVal(int doc) throws IOException {
// make sure it has minX and area
- if (validX.get(doc)) {
- assert validY.get(doc);
- return calculator.distance(from, Double.longBitsToDouble(ptX.get(doc)), Double.longBitsToDouble(ptY.get(doc))) * multiplier;
+ double x = getDocValue(ptX, doc);
+ if (ptX.docID() == doc) {
+ double y = getDocValue(ptY, doc);
+ assert ptY.docID() == doc;
+ return calculator.distance(from, x, y) * multiplier;
}
return nullValue;
}
@Override
- public String toString(int doc) {
+ public String toString(int doc) throws IOException {
return description() + "=" + floatVal(doc);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java
index 99a1f94..a5344b7 100644
--- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java
+++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java
@@ -132,12 +132,16 @@ final class GeoPointTermQueryConstantScoreWrapper <Q extends GeoPointMultiTermQu
if (preApproved.get(docId)) {
return true;
} else {
- sdv.setDocument(docId);
- int count = sdv.count();
- for (int i = 0; i < count; i++) {
- long hash = sdv.valueAt(i);
- if (termsEnum.postFilter(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash))) {
- return true;
+ if (docId > sdv.docID()) {
+ sdv.advance(docId);
+ }
+ if (docId == sdv.docID()) {
+ int count = sdv.docValueCount();
+ for (int i = 0; i < count; i++) {
+ long hash = sdv.nextValue();
+ if (termsEnum.postFilter(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash))) {
+ return true;
+ }
}
}
return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointDistanceComparator.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointDistanceComparator.java
index 869e3a3..2a522cc 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointDistanceComparator.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointDistanceComparator.java
@@ -26,11 +26,10 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorer;
-
-import org.apache.lucene.spatial3d.geom.GeoDistanceShape;
-import org.apache.lucene.spatial3d.geom.XYZBounds;
import org.apache.lucene.spatial3d.geom.DistanceStyle;
+import org.apache.lucene.spatial3d.geom.GeoDistanceShape;
import org.apache.lucene.spatial3d.geom.PlanetModel;
+import org.apache.lucene.spatial3d.geom.XYZBounds;
/**
* Compares documents by distance from an origin point, using a GeoDistanceShape to compute the distance
@@ -90,16 +89,19 @@ class Geo3DPointDistanceComparator extends FieldComparator<Double> implements Le
@Override
public int compareBottom(int doc) throws IOException {
- currentDocs.setDocument(doc);
-
- int numValues = currentDocs.count();
- if (numValues == 0) {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
+ if (doc < currentDocs.docID()) {
return Double.compare(bottomDistance, Double.POSITIVE_INFINITY);
}
+
+ int numValues = currentDocs.docValueCount();
+ assert numValues > 0;
int cmp = -1;
for (int i = 0; i < numValues; i++) {
- long encoded = currentDocs.valueAt(i);
+ long encoded = currentDocs.nextValue();
// Test against bounds.
// First we need to decode...
@@ -148,19 +150,22 @@ class Geo3DPointDistanceComparator extends FieldComparator<Double> implements Le
return Double.compare(topValue, computeMinimumDistance(doc));
}
- double computeMinimumDistance(final int doc) {
- currentDocs.setDocument(doc);
+ double computeMinimumDistance(final int doc) throws IOException {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
double minValue = Double.POSITIVE_INFINITY;
- final int numValues = currentDocs.count();
- for (int i = 0; i < numValues; i++) {
- final long encoded = currentDocs.valueAt(i);
- final double distance = distanceShape.computeDistance(DistanceStyle.ARC,
- Geo3DDocValuesField.decodeXValue(encoded),
- Geo3DDocValuesField.decodeYValue(encoded),
- Geo3DDocValuesField.decodeZValue(encoded));
- minValue = Math.min(minValue, distance);
+ if (doc == currentDocs.docID()) {
+ final int numValues = currentDocs.docValueCount();
+ for (int i = 0; i < numValues; i++) {
+ final long encoded = currentDocs.nextValue();
+ final double distance = distanceShape.computeDistance(DistanceStyle.ARC,
+ Geo3DDocValuesField.decodeXValue(encoded),
+ Geo3DDocValuesField.decodeYValue(encoded),
+ Geo3DDocValuesField.decodeZValue(encoded));
+ minValue = Math.min(minValue, distance);
+ }
}
return minValue;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointOutsideDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointOutsideDistanceComparator.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointOutsideDistanceComparator.java
index 10e0010..3ff8078 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointOutsideDistanceComparator.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPointOutsideDistanceComparator.java
@@ -26,9 +26,8 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorer;
-
-import org.apache.lucene.spatial3d.geom.GeoOutsideDistance;
import org.apache.lucene.spatial3d.geom.DistanceStyle;
+import org.apache.lucene.spatial3d.geom.GeoOutsideDistance;
import org.apache.lucene.spatial3d.geom.PlanetModel;
/**
@@ -70,16 +69,19 @@ class Geo3DPointOutsideDistanceComparator extends FieldComparator<Double> implem
@Override
public int compareBottom(int doc) throws IOException {
- currentDocs.setDocument(doc);
-
- int numValues = currentDocs.count();
- if (numValues == 0) {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
+ if (doc < currentDocs.docID()) {
return Double.compare(bottomDistance, Double.POSITIVE_INFINITY);
}
+ int numValues = currentDocs.docValueCount();
+ assert numValues > 0;
+
int cmp = -1;
for (int i = 0; i < numValues; i++) {
- long encoded = currentDocs.valueAt(i);
+ long encoded = currentDocs.nextValue();
// Test against bounds.
// First we need to decode...
@@ -119,19 +121,22 @@ class Geo3DPointOutsideDistanceComparator extends FieldComparator<Double> implem
return Double.compare(topValue, computeMinimumDistance(doc));
}
- double computeMinimumDistance(final int doc) {
- currentDocs.setDocument(doc);
+ double computeMinimumDistance(final int doc) throws IOException {
+ if (doc > currentDocs.docID()) {
+ currentDocs.advance(doc);
+ }
double minValue = Double.POSITIVE_INFINITY;
- final int numValues = currentDocs.count();
- for (int i = 0; i < numValues; i++) {
- final long encoded = currentDocs.valueAt(i);
- final double distance = distanceShape.computeOutsideDistance(DistanceStyle.ARC,
- Geo3DDocValuesField.decodeXValue(encoded),
- Geo3DDocValuesField.decodeYValue(encoded),
- Geo3DDocValuesField.decodeZValue(encoded));
- minValue = Math.min(minValue, distance);
+ if (doc == currentDocs.docID()) {
+ final int numValues = currentDocs.docValueCount();
+ for (int i = 0; i < numValues; i++) {
+ final long encoded = currentDocs.nextValue();
+ final double distance = distanceShape.computeOutsideDistance(DistanceStyle.ARC,
+ Geo3DDocValuesField.decodeXValue(encoded),
+ Geo3DDocValuesField.decodeYValue(encoded),
+ Geo3DDocValuesField.decodeZValue(encoded));
+ minValue = Math.min(minValue, distance);
+ }
}
return minValue;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
index d9baf61..450c526 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
@@ -791,8 +791,6 @@ public class TestGeo3DPoint extends LuceneTestCase {
final int iters = atLeast(100);
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
for (int iter=0;iter<iters;iter++) {
/*
@@ -835,8 +833,11 @@ public class TestGeo3DPoint extends LuceneTestCase {
System.err.println(" hitCount: " + hits.cardinality());
}
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
for(int docID=0;docID<r.maxDoc();docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
GeoPoint point = points[id];
GeoPoint unquantizedPoint = unquantizedPoints[id];
if (point != null && unquantizedPoint != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
index 9006ae2..a8dd507 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
@@ -239,12 +239,20 @@ public class DocumentDictionary implements Dictionary {
* or if it's indexed as {@link NumericDocValues} (using <code>docId</code>) for the document.
* If no value is found, then the weight is 0.
*/
- protected long getWeight(Document doc, int docId) {
+ protected long getWeight(Document doc, int docId) throws IOException {
IndexableField weight = doc.getField(weightField);
if (weight != null) { // found weight as stored
return (weight.numericValue() != null) ? weight.numericValue().longValue() : 0;
} else if (weightValues != null) { // found weight as NumericDocValue
- return weightValues.get(docId);
+ if (weightValues.docID() < docId) {
+ weightValues.advance(docId);
+ }
+ if (weightValues.docID() == docId) {
+ return weightValues.longValue();
+ } else {
+ // missing
+ return 0;
+ }
} else { // fall back
return 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
index 8affcc5..2c0b8f4 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
@@ -132,7 +132,7 @@ public class DocumentValueSourceDictionary extends DocumentDictionary {
* by the <code>weightsValueSource</code>
* */
@Override
- protected long getWeight(Document doc, int docId) {
+ protected long getWeight(Document doc, int docId) {
if (currentWeightValues == null) {
return 0;
}
@@ -145,7 +145,11 @@ public class DocumentValueSourceDictionary extends DocumentDictionary {
throw new RuntimeException(e);
}
}
- return currentWeightValues.longVal(docId - starts[subIndex]);
+ try {
+ return currentWeightValues.longVal(docId - starts[subIndex]);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index 9174137..d05c39f 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -620,22 +620,27 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
boolean doHighlight, Set<String> matchedTokens, String prefixToken)
throws IOException {
- BinaryDocValues textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
-
- // This will just be null if app didn't pass payloads to build():
- // TODO: maybe just stored fields? they compress...
- BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
List<LookupResult> results = new ArrayList<>();
for (int i=0;i<hits.scoreDocs.length;i++) {
FieldDoc fd = (FieldDoc) hits.scoreDocs[i];
- BytesRef term = textDV.get(fd.doc);
+ BinaryDocValues textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
+ textDV.advance(fd.doc);
+ BytesRef term = textDV.binaryValue();
String text = term.utf8ToString();
long score = (Long) fd.fields[0];
+ // This will just be null if app didn't pass payloads to build():
+ // TODO: maybe just stored fields? they compress...
+ BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
+
BytesRef payload;
if (payloadsDV != null) {
- payload = BytesRef.deepCopyOf(payloadsDV.get(fd.doc));
+ if (payloadsDV.advance(fd.doc) == fd.doc) {
+ payload = BytesRef.deepCopyOf(payloadsDV.binaryValue());
+ } else {
+ payload = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
} else {
payload = null;
}
@@ -646,11 +651,13 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
Set<BytesRef> contexts;
if (contextsDV != null) {
contexts = new HashSet<BytesRef>();
- contextsDV.setDocument(fd.doc - leaves.get(segment).docBase);
- long ord;
- while ((ord = contextsDV.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- BytesRef context = BytesRef.deepCopyOf(contextsDV.lookupOrd(ord));
- contexts.add(context);
+ int targetDocID = fd.doc - leaves.get(segment).docBase;
+ if (contextsDV.advance(targetDocID) == targetDocID) {
+ long ord;
+ while ((ord = contextsDV.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ BytesRef context = BytesRef.deepCopyOf(contextsDV.lookupOrd(ord));
+ contexts.add(context);
+ }
}
} else {
contexts = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
index 925f8b0..413d401 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
@@ -28,9 +28,9 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
@@ -186,13 +186,6 @@ public class BlendedInfixSuggester extends AnalyzingInfixSuggester {
boolean doHighlight, Set<String> matchedTokens, String prefixToken)
throws IOException {
- BinaryDocValues textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
- assert textDV != null;
-
- // This will just be null if app didn't pass payloads to build():
- // TODO: maybe just stored fields? they compress...
- BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
-
TreeSet<Lookup.LookupResult> results = new TreeSet<>(LOOKUP_COMP);
// we reduce the num to the one initially requested
@@ -201,12 +194,25 @@ public class BlendedInfixSuggester extends AnalyzingInfixSuggester {
for (int i = 0; i < hits.scoreDocs.length; i++) {
FieldDoc fd = (FieldDoc) hits.scoreDocs[i];
- final String text = textDV.get(fd.doc).utf8ToString();
+ BinaryDocValues textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
+ assert textDV != null;
+
+ textDV.advance(fd.doc);
+
+ final String text = textDV.binaryValue().utf8ToString();
long weight = (Long) fd.fields[0];
+ // This will just be null if app didn't pass payloads to build():
+ // TODO: maybe just stored fields? they compress...
+ BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
+
BytesRef payload;
if (payloadsDV != null) {
- payload = BytesRef.deepCopyOf(payloadsDV.get(fd.doc));
+ if (payloadsDV.advance(fd.doc) == fd.doc) {
+ payload = BytesRef.deepCopyOf(payloadsDV.binaryValue());
+ } else {
+ payload = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
} else {
payload = null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
index 777c0c8..20561b3 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestPrefixCompletionQuery.java
@@ -34,11 +34,13 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.suggest.BitsProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.apache.lucene.search.suggest.document.TestSuggestField.Entry;
import static org.apache.lucene.search.suggest.document.TestSuggestField.assertSuggestions;
import static org.apache.lucene.search.suggest.document.TestSuggestField.iwcWithSuggestField;
@@ -81,30 +83,21 @@ public class TestPrefixCompletionQuery extends LuceneTestCase {
@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
final int maxDoc = context.reader().maxDoc();
+ FixedBitSet bits = new FixedBitSet(maxDoc);
final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
- return new Bits() {
-
- @Override
- public boolean get(int doc) {
- values.setDocument(doc);
- final int count = values.count();
- for (int i = 0; i < count; ++i) {
- final long v = values.valueAt(i);
- if (v >= min && v <= max) {
- return true;
- }
+ int docID;
+ while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
+ final int count = values.docValueCount();
+ for (int i = 0; i < count; ++i) {
+ final long v = values.nextValue();
+ if (v >= min && v <= max) {
+ bits.set(docID);
+ break;
}
- return false;
}
-
- @Override
- public int length() {
- return maxDoc;
- }
-
- };
+ }
+ return bits;
}
-
}
public Directory dir;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
index e0af9a1..76ab1df 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
@@ -18,31 +18,28 @@ package org.apache.lucene.codecs.asserting;
import java.io.IOException;
import java.util.Collection;
-import java.util.Iterator;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.AssertingLeafReader;
-import org.apache.lucene.index.AssertingLeafReader.AssertingRandomAccessOrds;
-import org.apache.lucene.index.AssertingLeafReader.AssertingSortedSetDocValues;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Just like the default but with additional asserts.
*/
@@ -78,131 +75,138 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- int count = 0;
- for (Number v : values) {
- count++;
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ NumericDocValues values = valuesProducer.getNumeric(field);
+
+ int docID;
+ int lastDocID = -1;
+ while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
+ assert docID >= 0 && docID < maxDoc;
+ assert docID > lastDocID;
+ lastDocID = docID;
+ long value = values.longValue();
}
- assert count == maxDoc;
- TestUtil.checkIterator(values.iterator(), maxDoc, true);
- in.addNumericField(field, values);
+
+ in.addNumericField(field, valuesProducer);
}
@Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- int count = 0;
- for (BytesRef b : values) {
- assert b == null || b.isValid();
- count++;
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ BinaryDocValues values = valuesProducer.getBinary(field);
+
+ int docID;
+ int lastDocID = -1;
+ while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
+ assert docID >= 0 && docID < maxDoc;
+ assert docID > lastDocID;
+ lastDocID = docID;
+ BytesRef value = values.binaryValue();
+ assert value.isValid();
}
- assert count == maxDoc;
- TestUtil.checkIterator(values.iterator(), maxDoc, true);
- in.addBinaryField(field, values);
+
+ in.addBinaryField(field, valuesProducer);
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- int valueCount = 0;
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ SortedDocValues values = valuesProducer.getSorted(field);
+
+ int valueCount = values.getValueCount();
+ assert valueCount <= maxDoc;
BytesRef lastValue = null;
- for (BytesRef b : values) {
+ for (int ord=0;ord<valueCount;ord++) {
+ BytesRef b = values.lookupOrd(ord);
assert b != null;
assert b.isValid();
- if (valueCount > 0) {
+ if (ord > 0) {
assert b.compareTo(lastValue) > 0;
}
lastValue = BytesRef.deepCopyOf(b);
- valueCount++;
}
- assert valueCount <= maxDoc;
FixedBitSet seenOrds = new FixedBitSet(valueCount);
- int count = 0;
- for (Number v : docToOrd) {
- assert v != null;
- int ord = v.intValue();
- assert ord >= -1 && ord < valueCount;
- if (ord >= 0) {
- seenOrds.set(ord);
- }
- count++;
+ int docID;
+ int lastDocID = -1;
+ while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
+ assert docID >= 0 && docID < maxDoc;
+ assert docID > lastDocID;
+ lastDocID = docID;
+ int ord = values.ordValue();
+ assert ord >= 0 && ord < valueCount;
+ seenOrds.set(ord);
}
- assert count == maxDoc;
assert seenOrds.cardinality() == valueCount;
- TestUtil.checkIterator(values.iterator(), valueCount, false);
- TestUtil.checkIterator(docToOrd.iterator(), maxDoc, false);
- in.addSortedField(field, values, docToOrd);
+ in.addSortedField(field, valuesProducer);
}
@Override
- public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+
long valueCount = 0;
- Iterator<Number> valueIterator = values.iterator();
- for (Number count : docToValueCount) {
- assert count != null;
- assert count.intValue() >= 0;
- valueCount += count.intValue();
+ int lastDocID = -1;
+ while (true) {
+ int docID = values.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
+ assert values.docID() > lastDocID;
+ lastDocID = values.docID();
+ int count = values.docValueCount();
+ assert count > 0;
+ valueCount += count;
long previous = Long.MIN_VALUE;
- for (int i = 0; i < count.intValue(); i++) {
- assert valueIterator.hasNext();
- Number next = valueIterator.next();
- assert next != null;
- long nextValue = next.longValue();
+ for (int i = 0; i < count; i++) {
+ long nextValue = values.nextValue();
assert nextValue >= previous;
previous = nextValue;
}
}
- assert valueIterator.hasNext() == false;
- TestUtil.checkIterator(docToValueCount.iterator(), maxDoc, false);
- TestUtil.checkIterator(values.iterator(), valueCount, false);
- in.addSortedNumericField(field, docToValueCount, values);
+ in.addSortedNumericField(field, valuesProducer);
}
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
- long valueCount = 0;
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+ SortedSetDocValues values = valuesProducer.getSortedSet(field);
+
+ long valueCount = values.getValueCount();
BytesRef lastValue = null;
- for (BytesRef b : values) {
+ for (long i=0;i<valueCount;i++) {
+ BytesRef b = values.lookupOrd(i);
assert b != null;
assert b.isValid();
- if (valueCount > 0) {
+ if (i > 0) {
assert b.compareTo(lastValue) > 0;
}
lastValue = BytesRef.deepCopyOf(b);
- valueCount++;
}
int docCount = 0;
- long ordCount = 0;
LongBitSet seenOrds = new LongBitSet(valueCount);
- Iterator<Number> ordIterator = ords.iterator();
- for (Number v : docToOrdCount) {
- assert v != null;
- int count = v.intValue();
- assert count >= 0;
+ while (true) {
+ int docID = values.nextDoc();
+ if (docID == NO_MORE_DOCS) {
+ break;
+ }
docCount++;
- ordCount += count;
long lastOrd = -1;
- for (int i = 0; i < count; i++) {
- Number o = ordIterator.next();
- assert o != null;
- long ord = o.longValue();
- assert ord >= 0 && ord < valueCount;
+ while (true) {
+ long ord = values.nextOrd();
+ if (ord == SortedSetDocValues.NO_MORE_ORDS) {
+ break;
+ }
+ assert ord >= 0 && ord < valueCount: "ord=" + ord + " is not in bounds 0 .." + (valueCount-1);
assert ord > lastOrd : "ord=" + ord + ",lastOrd=" + lastOrd;
seenOrds.set(ord);
lastOrd = ord;
}
}
- assert ordIterator.hasNext() == false;
- assert docCount == maxDoc;
assert seenOrds.cardinality() == valueCount;
- TestUtil.checkIterator(values.iterator(), valueCount, false);
- TestUtil.checkIterator(docToOrdCount.iterator(), maxDoc, false);
- TestUtil.checkIterator(ords.iterator(), ordCount, false);
- in.addSortedSetField(field, values, docToOrdCount, ords);
+ in.addSortedSetField(field, valuesProducer);
}
@Override
@@ -262,23 +266,10 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
assert field.getDocValuesType() == DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
assert values != null;
- if (values instanceof RandomAccessOrds) {
- return new AssertingRandomAccessOrds((RandomAccessOrds) values, maxDoc);
- } else {
- return new AssertingSortedSetDocValues(values, maxDoc);
- }
+ return new AssertingLeafReader.AssertingSortedSetDocValues(values, maxDoc);
}
@Override
- public Bits getDocsWithField(FieldInfo field) throws IOException {
- assert field.getDocValuesType() != DocValuesType.NONE;
- Bits bits = in.getDocsWithField(field);
- assert bits != null;
- assert bits.length() == maxDoc;
- return new AssertingLeafReader.AssertingBits(bits);
- }
-
- @Override
public void close() throws IOException {
in.close();
in.close(); // close again
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
index 4837513..f4abb54 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
@@ -106,7 +106,7 @@ public class AssertingLiveDocsFormat extends LiveDocsFormat {
@Override
public boolean get(int index) {
assert index >= 0;
- assert index < in.length();
+ assert index < in.length(): "index=" + index + " vs in.length()=" + in.length();
return in.get(index);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
index e646eb5..bf830bf 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
@@ -30,6 +30,8 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* Just like the default but with additional asserts.
*/
@@ -61,15 +63,19 @@ public class AssertingNormsFormat extends NormsFormat {
}
@Override
- public void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
- int count = 0;
- for (Number v : values) {
- assert v != null;
- count++;
+ public void addNormsField(FieldInfo field, NormsProducer valuesProducer) throws IOException {
+ NumericDocValues values = valuesProducer.getNorms(field);
+
+ int docID;
+ int lastDocID = -1;
+ while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
+ assert docID >= 0 && docID < maxDoc;
+ assert docID > lastDocID;
+ lastDocID = docID;
+ long value = values.longValue();
}
- assert count == maxDoc;
- TestUtil.checkIterator(values.iterator(), maxDoc, false);
- in.addNormsField(field, values);
+
+ in.addNormsField(field, valuesProducer);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
index f29a3f9..10ae536 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
@@ -25,7 +25,6 @@ import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
class CrankyDocValuesFormat extends DocValuesFormat {
final DocValuesFormat delegate;
@@ -70,43 +69,43 @@ class CrankyDocValuesFormat extends DocValuesFormat {
}
@Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from DocValuesConsumer.addNumericField()");
}
- delegate.addNumericField(field, values);
+ delegate.addNumericField(field, valuesProducer);
}
@Override
- public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from DocValuesConsumer.addBinaryField()");
}
- delegate.addBinaryField(field, values);
+ delegate.addBinaryField(field, valuesProducer);
}
@Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from DocValuesConsumer.addSortedField()");
}
- delegate.addSortedField(field, values, docToOrd);
+ delegate.addSortedField(field, valuesProducer);
}
@Override
- public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from DocValuesConsumer.addSortedNumericField()");
}
- delegate.addSortedNumericField(field, docToValueCount, values);
+ delegate.addSortedNumericField(field, valuesProducer);
}
@Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from DocValuesConsumer.addSortedSetField()");
}
- delegate.addSortedSetField(field, values, docToOrdCount, ords);
+ delegate.addSortedSetField(field, valuesProducer);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyNormsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyNormsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyNormsFormat.java
index 34b606a..9168ce0 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyNormsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyNormsFormat.java
@@ -66,11 +66,11 @@ class CrankyNormsFormat extends NormsFormat {
}
@Override
- public void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
+ public void addNormsField(FieldInfo field, NormsProducer valuesProducer) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from NormsConsumer.addNormsField()");
}
- delegate.addNormsField(field, values);
+ delegate.addNormsField(field, valuesProducer);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java
index 275c186..19b4a1c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java
@@ -37,9 +37,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
-import org.apache.lucene.geo.Rectangle;
-import org.apache.lucene.geo.GeoUtils;
-import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -792,8 +789,6 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
int iters = atLeast(25);
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
@@ -833,8 +828,10 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
});
boolean fail = false;
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for(int docID=0;docID<maxDoc;docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
@@ -920,8 +917,6 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
int iters = atLeast(25);
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
@@ -971,8 +966,10 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
});
boolean fail = false;
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for(int docID=0;docID<maxDoc;docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
@@ -1062,8 +1059,6 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
final int iters = atLeast(75);
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
@@ -1103,8 +1098,10 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
});
boolean fail = false;
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for(int docID=0;docID<maxDoc;docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected;
if (liveDocs != null && liveDocs.get(docID) == false) {
// document is deleted
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index c33db57..a109eb8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -396,18 +396,64 @@ public class AssertingLeafReader extends FilterLeafReader {
private final Thread creationThread = Thread.currentThread();
private final NumericDocValues in;
private final int maxDoc;
+ private int lastDocID = -1;
public AssertingNumericDocValues(NumericDocValues in, int maxDoc) {
this.in = in;
this.maxDoc = maxDoc;
+ // should start unpositioned:
+ assert in.docID() == -1;
}
@Override
- public long get(int docID) {
+ public int docID() {
+ assertThread("Numeric doc values", creationThread);
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assertThread("Numeric doc values", creationThread);
+ int docID = in.nextDoc();
+ assert docID > lastDocID;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ assert docID == in.docID();
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
assertThread("Numeric doc values", creationThread);
- assert docID >= 0 && docID < maxDoc;
- return in.get(docID);
+ assert target >= 0;
+ assert target >= in.docID();
+ int docID = in.advance(target);
+ assert docID >= target;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ assertThread("Numeric doc values", creationThread);
+ long cost = in.cost();
+ assert cost >= 0;
+ return cost;
+ }
+
+ @Override
+ public long longValue() {
+ assertThread("Numeric doc values", creationThread);
+ assert in.docID() != -1;
+ assert in.docID() != NO_MORE_DOCS;
+ return in.longValue();
}
+
+ @Override
+ public String toString() {
+ return "AssertingNumericDocValues(" + in + ")";
+ }
}
/** Wraps a BinaryDocValues but with additional asserts */
@@ -415,28 +461,73 @@ public class AssertingLeafReader extends FilterLeafReader {
private final Thread creationThread = Thread.currentThread();
private final BinaryDocValues in;
private final int maxDoc;
+ private int lastDocID = -1;
public AssertingBinaryDocValues(BinaryDocValues in, int maxDoc) {
this.in = in;
this.maxDoc = maxDoc;
+ // should start unpositioned:
+ assert in.docID() == -1;
}
@Override
- public BytesRef get(int docID) {
+ public int docID() {
assertThread("Binary doc values", creationThread);
- assert docID >= 0 && docID < maxDoc;
- final BytesRef result = in.get(docID);
- assert result.isValid();
- return result;
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assertThread("Binary doc values", creationThread);
+ int docID = in.nextDoc();
+ assert docID > lastDocID;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ assert docID == in.docID();
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ assertThread("Binary doc values", creationThread);
+ assert target >= 0;
+ assert target >= in.docID();
+ int docID = in.advance(target);
+ assert docID >= target;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ assertThread("Binary doc values", creationThread);
+ long cost = in.cost();
+ assert cost >= 0;
+ return cost;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ assertThread("Binary doc values", creationThread);
+ assert in.docID() != -1;
+ assert in.docID() != NO_MORE_DOCS;
+ return in.binaryValue();
+ }
+
+ @Override
+ public String toString() {
+ return "AssertingBinaryDocValues(" + in + ")";
}
}
-
+
/** Wraps a SortedDocValues but with additional asserts */
public static class AssertingSortedDocValues extends SortedDocValues {
private final Thread creationThread = Thread.currentThread();
private final SortedDocValues in;
private final int maxDoc;
private final int valueCount;
+ private int lastDocID = -1;
public AssertingSortedDocValues(SortedDocValues in, int maxDoc) {
this.in = in;
@@ -446,10 +537,46 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public int getOrd(int docID) {
+ public int docID() {
assertThread("Sorted doc values", creationThread);
- assert docID >= 0 && docID < maxDoc;
- int ord = in.getOrd(docID);
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assertThread("Sorted doc values", creationThread);
+ int docID = in.nextDoc();
+ assert docID > lastDocID;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ assert docID == in.docID();
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ assertThread("Sorted doc values", creationThread);
+ assert target >= 0;
+ assert target >= in.docID();
+ int docID = in.advance(target);
+ assert docID >= target;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ lastDocID = docID;
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ assertThread("Sorted doc values", creationThread);
+ long cost = in.cost();
+ assert cost >= 0;
+ return cost;
+ }
+
+ @Override
+ public int ordValue() {
+ assertThread("Sorted doc values", creationThread);
+ int ord = in.ordValue();
assert ord >= -1 && ord < valueCount;
return ord;
}
@@ -472,10 +599,9 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public BytesRef get(int docID) {
+ public BytesRef binaryValue() {
assertThread("Sorted doc values", creationThread);
- assert docID >= 0 && docID < maxDoc;
- final BytesRef result = in.get(docID);
+ final BytesRef result = in.binaryValue();
assert result.isValid();
return result;
}
@@ -491,11 +617,13 @@ public class AssertingLeafReader extends FilterLeafReader {
}
}
- /** Wraps a SortedSetDocValues but with additional asserts */
+ /** Wraps a SortedNumericDocValues but with additional asserts */
public static class AssertingSortedNumericDocValues extends SortedNumericDocValues {
private final Thread creationThread = Thread.currentThread();
private final SortedNumericDocValues in;
private final int maxDoc;
+ private int lastDocID = -1;
+ private int valueUpto;
public AssertingSortedNumericDocValues(SortedNumericDocValues in, int maxDoc) {
this.in = in;
@@ -503,110 +631,58 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public void setDocument(int doc) {
- assertThread("Sorted numeric doc values", creationThread);
- assert doc >= 0 && doc < maxDoc;
- in.setDocument(doc);
- // check the values are actually sorted
- long previous = Long.MIN_VALUE;
- for (int i = 0; i < in.count(); i++) {
- long v = in.valueAt(i);
- assert v >= previous;
- previous = v;
- }
- }
-
- @Override
- public long valueAt(int index) {
- assertThread("Sorted numeric doc values", creationThread);
- assert index < in.count();
- return in.valueAt(index);
- }
-
- @Override
- public int count() {
- assertThread("Sorted numeric doc values", creationThread);
- return in.count();
- }
- }
-
- /** Wraps a RandomAccessOrds but with additional asserts */
- public static class AssertingRandomAccessOrds extends RandomAccessOrds {
- private final Thread creationThread = Thread.currentThread();
- private final RandomAccessOrds in;
- private final int maxDoc;
- private final long valueCount;
- long lastOrd = NO_MORE_ORDS;
-
- public AssertingRandomAccessOrds(RandomAccessOrds in, int maxDoc) {
- this.in = in;
- this.maxDoc = maxDoc;
- this.valueCount = in.getValueCount();
- assert valueCount >= 0;
- }
-
- @Override
- public long nextOrd() {
- assertThread("Sorted numeric doc values", creationThread);
- assert lastOrd != NO_MORE_ORDS;
- long ord = in.nextOrd();
- assert ord < valueCount;
- assert ord == NO_MORE_ORDS || ord > lastOrd;
- lastOrd = ord;
- return ord;
- }
-
- @Override
- public void setDocument(int docID) {
- assertThread("Sorted numeric doc values", creationThread);
- assert docID >= 0 && docID < maxDoc : "docid=" + docID + ",maxDoc=" + maxDoc;
- in.setDocument(docID);
- lastOrd = -2;
+ public int docID() {
+ return in.docID();
}
@Override
- public BytesRef lookupOrd(long ord) {
+ public int nextDoc() throws IOException {
assertThread("Sorted numeric doc values", creationThread);
- assert ord >= 0 && ord < valueCount;
- final BytesRef result = in.lookupOrd(ord);
- assert result.isValid();
- return result;
+ int docID = in.nextDoc();
+ assert docID > lastDocID;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ assert docID == in.docID();
+ lastDocID = docID;
+ valueUpto = 0;
+ return docID;
}
@Override
- public long getValueCount() {
+ public int advance(int target) throws IOException {
assertThread("Sorted numeric doc values", creationThread);
- long valueCount = in.getValueCount();
- assert valueCount == this.valueCount; // should not change
- return valueCount;
+ assert target >= 0;
+ assert target >= in.docID();
+ int docID = in.advance(target);
+ assert docID == in.docID();
+ assert docID >= target;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ lastDocID = docID;
+ valueUpto = 0;
+ return docID;
}
@Override
- public long lookupTerm(BytesRef key) {
+ public long cost() {
assertThread("Sorted numeric doc values", creationThread);
- assert key.isValid();
- long result = in.lookupTerm(key);
- assert result < valueCount;
- assert key.isValid();
- return result;
+ long cost = in.cost();
+ assert cost >= 0;
+ return cost;
}
@Override
- public long ordAt(int index) {
+ public long nextValue() throws IOException {
assertThread("Sorted numeric doc values", creationThread);
- assert index < in.cardinality();
- long ord = in.ordAt(index);
- assert ord >= 0 && ord < valueCount;
- return ord;
+ assert valueUpto < in.docValueCount(): "valueUpto=" + valueUpto + " in.docValueCount()=" + in.docValueCount();
+ valueUpto++;
+ return in.nextValue();
}
@Override
- public int cardinality() {
+ public int docValueCount() {
assertThread("Sorted numeric doc values", creationThread);
- int cardinality = in.cardinality();
- assert cardinality >= 0;
- return cardinality;
- }
+ assert in.docValueCount() > 0;
+ return in.docValueCount();
+ }
}
/** Wraps a SortedSetDocValues but with additional asserts */
@@ -615,6 +691,7 @@ public class AssertingLeafReader extends FilterLeafReader {
private final SortedSetDocValues in;
private final int maxDoc;
private final long valueCount;
+ private int lastDocID = -1;
long lastOrd = NO_MORE_ORDS;
public AssertingSortedSetDocValues(SortedSetDocValues in, int maxDoc) {
@@ -623,10 +700,50 @@ public class AssertingLeafReader extends FilterLeafReader {
this.valueCount = in.getValueCount();
assert valueCount >= 0;
}
+
+ @Override
+ public int docID() {
+ assertThread("Sorted set doc values", creationThread);
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ assertThread("Sorted set doc values", creationThread);
+ int docID = in.nextDoc();
+ assert docID > lastDocID;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ assert docID == in.docID();
+ lastDocID = docID;
+ lastOrd = -2;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ assertThread("Sorted set doc values", creationThread);
+ assert target >= 0;
+ assert target >= in.docID();
+ int docID = in.advance(target);
+ assert docID == in.docID();
+ assert docID >= target;
+ assert docID == NO_MORE_DOCS || docID < maxDoc;
+ lastDocID = docID;
+ lastOrd = -2;
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ assertThread("Sorted set doc values", creationThread);
+ long cost = in.cost();
+ assert cost >= 0;
+ return cost;
+ }
@Override
- public long nextOrd() {
- assertThread("Sorted numeric doc values", creationThread);
+ public long nextOrd() throws IOException {
+ assertThread("Sorted set doc values", creationThread);
assert lastOrd != NO_MORE_ORDS;
long ord = in.nextOrd();
assert ord < valueCount;
@@ -636,16 +753,8 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public void setDocument(int docID) {
- assertThread("Sorted numeric doc values", creationThread);
- assert docID >= 0 && docID < maxDoc : "docid=" + docID + ",maxDoc=" + maxDoc;
- in.setDocument(docID);
- lastOrd = -2;
- }
-
- @Override
public BytesRef lookupOrd(long ord) {
- assertThread("Sorted numeric doc values", creationThread);
+ assertThread("Sorted set doc values", creationThread);
assert ord >= 0 && ord < valueCount;
final BytesRef result = in.lookupOrd(ord);
assert result.isValid();
@@ -654,7 +763,7 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public long getValueCount() {
- assertThread("Sorted numeric doc values", creationThread);
+ assertThread("Sorted set doc values", creationThread);
long valueCount = in.getValueCount();
assert valueCount == this.valueCount; // should not change
return valueCount;
@@ -662,7 +771,7 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public long lookupTerm(BytesRef key) {
- assertThread("Sorted numeric doc values", creationThread);
+ assertThread("Sorted set doc values", creationThread);
assert key.isValid();
long result = in.lookupTerm(key);
assert result < valueCount;
@@ -734,11 +843,7 @@ public class AssertingLeafReader extends FilterLeafReader {
if (dv != null) {
assert fi != null;
assert fi.getDocValuesType() == DocValuesType.SORTED_SET;
- if (dv instanceof RandomAccessOrds) {
- return new AssertingRandomAccessOrds((RandomAccessOrds) dv, maxDoc());
- } else {
- return new AssertingSortedSetDocValues(dv, maxDoc());
- }
+ return new AssertingSortedSetDocValues(dv, maxDoc());
} else {
assert fi == null || fi.getDocValuesType() != DocValuesType.SORTED_SET;
return null;
@@ -795,21 +900,6 @@ public class AssertingLeafReader extends FilterLeafReader {
return liveDocs;
}
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- Bits docsWithField = super.getDocsWithField(field);
- FieldInfo fi = getFieldInfos().fieldInfo(field);
- if (docsWithField != null) {
- assert fi != null;
- assert fi.getDocValuesType() != DocValuesType.NONE;
- assert maxDoc() == docsWithField.length();
- docsWithField = new AssertingBits(docsWithField);
- } else {
- assert fi == null || fi.getDocValuesType() == DocValuesType.NONE;
- }
- return docsWithField;
- }
-
// we don't change behavior of the reader: just validate the API.
@Override
[11/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
index 735dfbe..b638a29 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -35,6 +34,10 @@ import org.apache.lucene.util.BytesRefBuilder;
* TopFieldCollector}. The concrete public FieldComparator
* classes here correspond to the SortField types.
*
+ * <p>The document IDs passed to these methods must only
+ * move forwards, since they are using doc values iterators
+ * to retrieve sort values.</p>
+ *
* <p>This API is designed to achieve high performance
* sorting, by exposing a tight interaction with {@link
* FieldValueHitQueue} as it visits hits. Whenever a hit is
@@ -140,7 +143,6 @@ public abstract class FieldComparator<T> {
public static abstract class NumericComparator<T extends Number> extends SimpleFieldComparator<T> {
protected final T missingValue;
protected final String field;
- protected Bits docsWithField;
protected NumericDocValues currentReaderValues;
public NumericComparator(String field, T missingValue) {
@@ -151,26 +153,12 @@ public abstract class FieldComparator<T> {
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
currentReaderValues = getNumericDocValues(context, field);
- if (missingValue != null) {
- docsWithField = getDocsWithValue(context, field);
- // optimization to remove unneeded checks on the bit interface:
- if (docsWithField instanceof Bits.MatchAllBits) {
- docsWithField = null;
- }
- } else {
- docsWithField = null;
- }
}
/** Retrieves the NumericDocValues for the field in this segment */
protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
return DocValues.getNumeric(context.reader(), field);
}
-
- /** Retrieves a {@link Bits} instance representing documents that have a value in this segment. */
- protected Bits getDocsWithValue(LeafReaderContext context, String field) throws IOException {
- return DocValues.getDocsWithField(context.reader(), field);
- }
}
/** Parses field's values as double (using {@link
@@ -182,40 +170,38 @@ public abstract class FieldComparator<T> {
/**
* Creates a new comparator based on {@link Double#compare} for {@code numHits}.
- * When a document has no value for the field, {@code missingValue} is substituted.
+ * When a document has no value for the field, {@code missingValue} is substituted.
*/
public DoubleComparator(int numHits, String field, Double missingValue) {
- super(field, missingValue);
+ super(field, missingValue != null ? missingValue : 0.0);
values = new double[numHits];
}
+ private double getValueForDoc(int doc) throws IOException {
+ int curDocID = currentReaderValues.docID();
+ if (doc > curDocID) {
+ curDocID = currentReaderValues.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Double.longBitsToDouble(currentReaderValues.longValue());
+ } else {
+ return missingValue;
+ }
+ }
+
@Override
public int compare(int slot1, int slot2) {
return Double.compare(values[slot1], values[slot2]);
}
@Override
- public int compareBottom(int doc) {
- double v2 = Double.longBitsToDouble(currentReaderValues.get(doc));
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- return Double.compare(bottom, v2);
+ public int compareBottom(int doc) throws IOException {
+ return Double.compare(bottom, getValueForDoc(doc));
}
@Override
- public void copy(int slot, int doc) {
- double v2 = Double.longBitsToDouble(currentReaderValues.get(doc));
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- values[slot] = v2;
+ public void copy(int slot, int doc) throws IOException {
+ values[slot] = getValueForDoc(doc);
}
@Override
@@ -234,14 +220,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
- double docValue = Double.longBitsToDouble(currentReaderValues.get(doc));
- // Test for docValue == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
- docValue = missingValue;
- }
- return Double.compare(topValue, docValue);
+ public int compareTop(int doc) throws IOException {
+ return Double.compare(topValue, getValueForDoc(doc));
}
}
@@ -257,38 +237,35 @@ public abstract class FieldComparator<T> {
* When a document has no value for the field, {@code missingValue} is substituted.
*/
public FloatComparator(int numHits, String field, Float missingValue) {
- super(field, missingValue);
+ super(field, missingValue != null ? missingValue : 0.0f);
values = new float[numHits];
}
+ private float getValueForDoc(int doc) throws IOException {
+ int curDocID = currentReaderValues.docID();
+ if (doc > curDocID) {
+ curDocID = currentReaderValues.advance(doc);
+ }
+ if (doc == curDocID) {
+ return Float.intBitsToFloat((int) currentReaderValues.longValue());
+ } else {
+ return missingValue;
+ }
+ }
+
@Override
public int compare(int slot1, int slot2) {
return Float.compare(values[slot1], values[slot2]);
}
@Override
- public int compareBottom(int doc) {
- // TODO: are there sneaky non-branch ways to compute sign of float?
- float v2 = Float.intBitsToFloat((int)currentReaderValues.get(doc));
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- return Float.compare(bottom, v2);
+ public int compareBottom(int doc) throws IOException {
+ return Float.compare(bottom, getValueForDoc(doc));
}
@Override
- public void copy(int slot, int doc) {
- float v2 = Float.intBitsToFloat((int)currentReaderValues.get(doc));
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- values[slot] = v2;
+ public void copy(int slot, int doc) throws IOException {
+ values[slot] = getValueForDoc(doc);
}
@Override
@@ -307,14 +284,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
- float docValue = Float.intBitsToFloat((int)currentReaderValues.get(doc));
- // Test for docValue == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
- docValue = missingValue;
- }
- return Float.compare(topValue, docValue);
+ public int compareTop(int doc) throws IOException {
+ return Float.compare(topValue, getValueForDoc(doc));
}
}
@@ -330,9 +301,23 @@ public abstract class FieldComparator<T> {
* When a document has no value for the field, {@code missingValue} is substituted.
*/
public IntComparator(int numHits, String field, Integer missingValue) {
- super(field, missingValue);
+ super(field, missingValue != null ? missingValue : 0);
+ //System.out.println("IntComparator.init");
+ //new Throwable().printStackTrace(System.out);
values = new int[numHits];
}
+
+ private int getValueForDoc(int doc) throws IOException {
+ int curDocID = currentReaderValues.docID();
+ if (doc > curDocID) {
+ curDocID = currentReaderValues.advance(doc);
+ }
+ if (doc == curDocID) {
+ return (int) currentReaderValues.longValue();
+ } else {
+ return missingValue;
+ }
+ }
@Override
public int compare(int slot1, int slot2) {
@@ -340,27 +325,13 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareBottom(int doc) {
- int v2 = (int) currentReaderValues.get(doc);
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- return Integer.compare(bottom, v2);
+ public int compareBottom(int doc) throws IOException {
+ return Integer.compare(bottom, getValueForDoc(doc));
}
@Override
- public void copy(int slot, int doc) {
- int v2 = (int) currentReaderValues.get(doc);
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- values[slot] = v2;
+ public void copy(int slot, int doc) throws IOException {
+ values[slot] = getValueForDoc(doc);
}
@Override
@@ -379,14 +350,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
- int docValue = (int) currentReaderValues.get(doc);
- // Test for docValue == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
- docValue = missingValue;
- }
- return Integer.compare(topValue, docValue);
+ public int compareTop(int doc) throws IOException {
+ return Integer.compare(topValue, getValueForDoc(doc));
}
}
@@ -402,39 +367,35 @@ public abstract class FieldComparator<T> {
* When a document has no value for the field, {@code missingValue} is substituted.
*/
public LongComparator(int numHits, String field, Long missingValue) {
- super(field, missingValue);
+ super(field, missingValue != null ? missingValue : 0L);
values = new long[numHits];
}
+ private long getValueForDoc(int doc) throws IOException {
+ int curDocID = currentReaderValues.docID();
+ if (doc > curDocID) {
+ curDocID = currentReaderValues.advance(doc);
+ }
+ if (doc == curDocID) {
+ return currentReaderValues.longValue();
+ } else {
+ return missingValue;
+ }
+ }
+
@Override
public int compare(int slot1, int slot2) {
return Long.compare(values[slot1], values[slot2]);
}
@Override
- public int compareBottom(int doc) {
- // TODO: there are sneaky non-branch ways to compute
- // -1/+1/0 sign
- long v2 = currentReaderValues.get(doc);
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- return Long.compare(bottom, v2);
+ public int compareBottom(int doc) throws IOException {
+ return Long.compare(bottom, getValueForDoc(doc));
}
@Override
- public void copy(int slot, int doc) {
- long v2 = currentReaderValues.get(doc);
- // Test for v2 == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
- v2 = missingValue;
- }
-
- values[slot] = v2;
+ public void copy(int slot, int doc) throws IOException {
+ values[slot] = getValueForDoc(doc);
}
@Override
@@ -453,14 +414,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
- long docValue = currentReaderValues.get(doc);
- // Test for docValue == 0 to save Bits.get method call for
- // the common case (doc has value and value is non-zero):
- if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
- docValue = missingValue;
- }
- return Long.compare(topValue, docValue);
+ public int compareTop(int doc) throws IOException {
+ return Long.compare(topValue, getValueForDoc(doc));
}
}
@@ -700,6 +655,18 @@ public abstract class FieldComparator<T> {
}
}
+ private int getOrdForDoc(int doc) throws IOException {
+ int curDocID = termsIndex.docID();
+ if (doc > curDocID) {
+ if (termsIndex.advance(doc) == doc) {
+ return termsIndex.ordValue();
+ }
+ } else if (doc == curDocID) {
+ return termsIndex.ordValue();
+ }
+ return -1;
+ }
+
@Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
@@ -720,9 +687,9 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareBottom(int doc) {
+ public int compareBottom(int doc) throws IOException {
assert bottomSlot != -1;
- int docOrd = termsIndex.getOrd(doc);
+ int docOrd = getOrdForDoc(doc);
if (docOrd == -1) {
docOrd = missingOrd;
}
@@ -740,8 +707,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public void copy(int slot, int doc) {
- int ord = termsIndex.getOrd(doc);
+ public void copy(int slot, int doc) throws IOException {
+ int ord = getOrdForDoc(doc);
if (ord == -1) {
ord = missingOrd;
values[slot] = null;
@@ -836,9 +803,9 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
+ public int compareTop(int doc) throws IOException {
- int ord = termsIndex.getOrd(doc);
+ int ord = getOrdForDoc(doc);
if (ord == -1) {
ord = missingOrd;
}
@@ -883,7 +850,6 @@ public abstract class FieldComparator<T> {
private final BytesRef[] values;
private final BytesRefBuilder[] tempBRs;
private BinaryDocValues docTerms;
- private Bits docsWithField;
private final String field;
private BytesRef bottom;
private BytesRef topValue;
@@ -897,6 +863,18 @@ public abstract class FieldComparator<T> {
missingSortCmp = sortMissingLast ? 1 : -1;
}
+ private BytesRef getValueForDoc(int doc) throws IOException {
+ int curDocID = docTerms.docID();
+ if (doc > curDocID) {
+ curDocID = docTerms.advance(doc);
+ }
+ if (doc == curDocID) {
+ return docTerms.binaryValue();
+ } else {
+ return null;
+ }
+ }
+
@Override
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
@@ -905,14 +883,14 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareBottom(int doc) {
- final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
+ public int compareBottom(int doc) throws IOException {
+ final BytesRef comparableBytes = getValueForDoc(doc);
return compareValues(bottom, comparableBytes);
}
@Override
- public void copy(int slot, int doc) {
- final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
+ public void copy(int slot, int doc) throws IOException {
+ final BytesRef comparableBytes = getValueForDoc(doc);
if (comparableBytes == null) {
values[slot] = null;
} else {
@@ -929,27 +907,17 @@ public abstract class FieldComparator<T> {
return DocValues.getBinary(context.reader(), field);
}
- /** Retrieves the set of documents that have a value in this segment */
- protected Bits getDocsWithField(LeafReaderContext context, String field) throws IOException {
- return DocValues.getDocsWithField(context.reader(), field);
- }
-
/** Check whether the given value represents <tt>null</tt>. This can be
* useful if the {@link BinaryDocValues} returned by {@link #getBinaryDocValues}
- * use a special value as a sentinel. The default implementation checks
- * {@link #getDocsWithField}.
+ * use a special value as a sentinel.
* <p>NOTE: The null value can only be an EMPTY {@link BytesRef}. */
- protected boolean isNull(int doc, BytesRef term) {
- return docsWithField != null && docsWithField.get(doc) == false;
+ protected boolean isNull(int doc, BytesRef term) throws IOException {
+ return getValueForDoc(doc) == null;
}
@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
docTerms = getBinaryDocValues(context, field);
- docsWithField = getDocsWithField(context, field);
- if (docsWithField instanceof Bits.MatchAllBits) {
- docsWithField = null;
- }
return this;
}
@@ -985,20 +953,8 @@ public abstract class FieldComparator<T> {
}
@Override
- public int compareTop(int doc) {
- final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
- return compareValues(topValue, comparableBytes);
- }
-
- /**
- * Given a document and a term, return the term itself if it exists or
- * <tt>null</tt> otherwise.
- */
- private BytesRef getComparableBytes(int doc, BytesRef term) {
- if (term.length == 0 && isNull(doc, term)) {
- return null;
- }
- return term;
+ public int compareTop(int doc) throws IOException {
+ return compareValues(topValue, getValueForDoc(doc));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java
index 42fc256..8a47eea 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java
@@ -20,13 +20,15 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Objects;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.util.Bits;
/**
* A {@link Query} that matches documents that have a value for a given field
- * as reported by {@link LeafReader#getDocsWithField(String)}.
+ * as reported by doc values iterators.
*/
public final class FieldValueQuery extends Query {
@@ -60,14 +62,41 @@ public final class FieldValueQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
- return new RandomAccessWeight(this, boost) {
-
+ return new ConstantScoreWeight(this, boost) {
@Override
- protected Bits getMatchingDocs(LeafReaderContext context) throws IOException {
- return context.reader().getDocsWithField(field);
- }
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ FieldInfos fieldInfos = context.reader().getFieldInfos();
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ if (fieldInfo == null) {
+ return null;
+ }
+ DocValuesType dvType = fieldInfo.getDocValuesType();
+ LeafReader reader = context.reader();
+ DocIdSetIterator iterator;
+ switch(dvType) {
+ case NONE:
+ return null;
+ case NUMERIC:
+ iterator = reader.getNumericDocValues(field);
+ break;
+ case BINARY:
+ iterator = reader.getBinaryDocValues(field);
+ break;
+ case SORTED:
+ iterator = reader.getSortedDocValues(field);
+ break;
+ case SORTED_NUMERIC:
+ iterator = reader.getSortedNumericDocValues(field);
+ break;
+ case SORTED_SET:
+ iterator = reader.getSortedSetDocValues(field);
+ break;
+ default:
+ throw new AssertionError();
+ }
+ return new ConstantScoreScorer(this, score(), iterator);
+ }
};
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
index acc8282..8165204 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
@@ -553,7 +553,7 @@ final class SloppyPhraseScorer extends Scorer {
}
@Override
- public float score() {
+ public float score() throws IOException {
return docScorer.score(docID(), sloppyFreq);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/SortedNumericSelector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSelector.java b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSelector.java
index 314cc4f..56626e8 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSelector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSelector.java
@@ -17,7 +17,10 @@
package org.apache.lucene.search;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FilterNumericDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.util.NumericUtils;
@@ -78,17 +81,17 @@ public class SortedNumericSelector {
// undo the numericutils sortability
switch(numericType) {
case FLOAT:
- return new NumericDocValues() {
+ return new FilterNumericDocValues(view) {
@Override
- public long get(int docID) {
- return NumericUtils.sortableFloatBits((int) view.get(docID));
+ public long longValue() {
+ return NumericUtils.sortableFloatBits((int) in.longValue());
}
};
case DOUBLE:
- return new NumericDocValues() {
+ return new FilterNumericDocValues(view) {
@Override
- public long get(int docID) {
- return NumericUtils.sortableDoubleBits(view.get(docID));
+ public long longValue() {
+ return NumericUtils.sortableDoubleBits(in.longValue());
}
};
default:
@@ -99,39 +102,93 @@ public class SortedNumericSelector {
/** Wraps a SortedNumericDocValues and returns the first value (min) */
static class MinValue extends NumericDocValues {
final SortedNumericDocValues in;
+ private long value;
MinValue(SortedNumericDocValues in) {
this.in = in;
}
@Override
- public long get(int docID) {
- in.setDocument(docID);
- if (in.count() == 0) {
- return 0; // missing
- } else {
- return in.valueAt(0);
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ int docID = in.nextDoc();
+ if (docID != NO_MORE_DOCS) {
+ value = in.nextValue();
}
+ return docID;
}
- }
-
+
+ @Override
+ public int advance(int target) throws IOException {
+ int docID = in.advance(target);
+ if (docID != NO_MORE_DOCS) {
+ value = in.nextValue();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
+ }
+
/** Wraps a SortedNumericDocValues and returns the last value (max) */
static class MaxValue extends NumericDocValues {
final SortedNumericDocValues in;
+ private long value;
MaxValue(SortedNumericDocValues in) {
this.in = in;
}
@Override
- public long get(int docID) {
- in.setDocument(docID);
- final int count = in.count();
- if (count == 0) {
- return 0; // missing
- } else {
- return in.valueAt(count-1);
+ public int docID() {
+ return in.docID();
+ }
+
+ private void setValue() throws IOException {
+ int count = in.docValueCount();
+ for(int i=0;i<count;i++) {
+ value = in.nextValue();
}
}
- }
+
+ @Override
+ public int nextDoc() throws IOException {
+ int docID = in.nextDoc();
+ if (docID != NO_MORE_DOCS) {
+ setValue();
+ }
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ int docID = in.advance(target);
+ if (docID != NO_MORE_DOCS) {
+ setValue();
+ }
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public long longValue() {
+ return value;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
index 188a408..22d7457 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
@@ -19,8 +19,8 @@ package org.apache.lucene.search;
import java.io.IOException;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java b/lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java
index dee5e26..25fa8d3 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java
@@ -17,12 +17,16 @@
package org.apache.lucene.search;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+
/** Selects a value from the document's set to use as the representative value */
public class SortedSetSelector {
@@ -73,18 +77,12 @@ public class SortedSetSelector {
// so just sort on the underlying single-valued dv directly.
// regardless of selector type, this optimization is safe!
return singleton;
- } else if (selector == Type.MIN) {
- return new MinValue(sortedSet);
} else {
- if (sortedSet instanceof RandomAccessOrds == false) {
- throw new UnsupportedOperationException("codec does not support random access ordinals, cannot use selector: " + selector + " docValsImpl: " + sortedSet.toString());
- }
- RandomAccessOrds randomOrds = (RandomAccessOrds) sortedSet;
switch(selector) {
- case MAX: return new MaxValue(randomOrds);
- case MIDDLE_MIN: return new MiddleMinValue(randomOrds);
- case MIDDLE_MAX: return new MiddleMaxValue(randomOrds);
- case MIN:
+ case MIN: return new MinValue(sortedSet);
+ case MAX: return new MaxValue(sortedSet);
+ case MIDDLE_MIN: return new MiddleMinValue(sortedSet);
+ case MIDDLE_MAX: return new MiddleMaxValue(sortedSet);
default:
throw new AssertionError();
}
@@ -94,15 +92,39 @@ public class SortedSetSelector {
/** Wraps a SortedSetDocValues and returns the first ordinal (min) */
static class MinValue extends SortedDocValues {
final SortedSetDocValues in;
+ private int ord;
MinValue(SortedSetDocValues in) {
this.in = in;
}
@Override
- public int getOrd(int docID) {
- in.setDocument(docID);
- return (int) in.nextOrd();
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ in.nextDoc();
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ in.advance(target);
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
}
@Override
@@ -119,25 +141,52 @@ public class SortedSetSelector {
public int lookupTerm(BytesRef key) {
return (int) in.lookupTerm(key);
}
+
+ private void setOrd() throws IOException {
+ if (docID() != NO_MORE_DOCS) {
+ ord = (int) in.nextOrd();
+ } else {
+ ord = (int) NO_MORE_ORDS;
+ }
+ }
}
/** Wraps a SortedSetDocValues and returns the last ordinal (max) */
static class MaxValue extends SortedDocValues {
- final RandomAccessOrds in;
+ final SortedSetDocValues in;
+ private int ord;
- MaxValue(RandomAccessOrds in) {
+ MaxValue(SortedSetDocValues in) {
this.in = in;
}
@Override
- public int getOrd(int docID) {
- in.setDocument(docID);
- final int count = in.cardinality();
- if (count == 0) {
- return -1;
- } else {
- return (int) in.ordAt(count-1);
- }
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ in.nextDoc();
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ in.advance(target);
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
}
@Override
@@ -149,30 +198,64 @@ public class SortedSetSelector {
public int getValueCount() {
return (int) in.getValueCount();
}
-
+
@Override
public int lookupTerm(BytesRef key) {
return (int) in.lookupTerm(key);
}
+
+ private void setOrd() throws IOException {
+ if (docID() != NO_MORE_DOCS) {
+ while(true) {
+ long nextOrd = in.nextOrd();
+ if (nextOrd == NO_MORE_ORDS) {
+ break;
+ }
+ ord = (int) nextOrd;
+ }
+ } else {
+ ord = (int) NO_MORE_ORDS;
+ }
+ }
}
/** Wraps a SortedSetDocValues and returns the middle ordinal (or min of the two) */
static class MiddleMinValue extends SortedDocValues {
- final RandomAccessOrds in;
+ final SortedSetDocValues in;
+ private int ord;
+ private int[] ords = new int[8];
- MiddleMinValue(RandomAccessOrds in) {
+ MiddleMinValue(SortedSetDocValues in) {
this.in = in;
}
@Override
- public int getOrd(int docID) {
- in.setDocument(docID);
- final int count = in.cardinality();
- if (count == 0) {
- return -1;
- } else {
- return (int) in.ordAt((count-1) >>> 1);
- }
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ in.nextDoc();
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ in.advance(target);
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
}
@Override
@@ -184,30 +267,76 @@ public class SortedSetSelector {
public int getValueCount() {
return (int) in.getValueCount();
}
-
+
@Override
public int lookupTerm(BytesRef key) {
return (int) in.lookupTerm(key);
}
+
+ private void setOrd() throws IOException {
+ if (docID() != NO_MORE_DOCS) {
+ int upto = 0;
+ while (true) {
+ long nextOrd = in.nextOrd();
+ if (nextOrd == NO_MORE_ORDS) {
+ break;
+ }
+ if (upto == ords.length) {
+ ords = ArrayUtil.grow(ords);
+ }
+ ords[upto++] = (int) nextOrd;
+ }
+
+ if (upto == 0) {
+ // iterator should not have returned this docID if it has no ords:
+ assert false;
+ ord = (int) NO_MORE_ORDS;
+ } else {
+ ord = ords[(upto-1) >>> 1];
+ }
+ } else {
+ ord = (int) NO_MORE_ORDS;
+ }
+ }
}
/** Wraps a SortedSetDocValues and returns the middle ordinal (or max of the two) */
static class MiddleMaxValue extends SortedDocValues {
- final RandomAccessOrds in;
+ final SortedSetDocValues in;
+ private int ord;
+ private int[] ords = new int[8];
- MiddleMaxValue(RandomAccessOrds in) {
+ MiddleMaxValue(SortedSetDocValues in) {
this.in = in;
}
@Override
- public int getOrd(int docID) {
- in.setDocument(docID);
- final int count = in.cardinality();
- if (count == 0) {
- return -1;
- } else {
- return (int) in.ordAt(count >>> 1);
- }
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ in.nextDoc();
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ in.advance(target);
+ setOrd();
+ return docID();
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ @Override
+ public int ordValue() {
+ return ord;
}
@Override
@@ -219,10 +348,36 @@ public class SortedSetSelector {
public int getValueCount() {
return (int) in.getValueCount();
}
-
+
@Override
public int lookupTerm(BytesRef key) {
return (int) in.lookupTerm(key);
}
+
+ private void setOrd() throws IOException {
+ if (docID() != NO_MORE_DOCS) {
+ int upto = 0;
+ while (true) {
+ long nextOrd = in.nextOrd();
+ if (nextOrd == NO_MORE_ORDS) {
+ break;
+ }
+ if (upto == ords.length) {
+ ords = ArrayUtil.grow(ords);
+ }
+ ords[upto++] = (int) nextOrd;
+ }
+
+ if (upto == 0) {
+ // iterator should not have returned this docID if it has no ords:
+ assert false;
+ ord = (int) NO_MORE_ORDS;
+ } else {
+ ord = ords[upto >>> 1];
+ }
+ } else {
+ ord = (int) NO_MORE_ORDS;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
index 5f81e85..da2546f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
@@ -19,8 +19,8 @@ package org.apache.lucene.search;
import java.io.IOException;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -122,8 +122,7 @@ public class SortedSetSortField extends SortField {
return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {
@Override
protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
- SortedSetDocValues sortedSet = DocValues.getSortedSet(context.reader(), field);
- return SortedSetSelector.wrap(sortedSet, selector);
+ return SortedSetSelector.wrap(DocValues.getSortedSet(context.reader(), field), selector);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/package-info.java b/lucene/core/src/java/org/apache/lucene/search/package-info.java
index b4d4780..fc0ab2e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/search/package-info.java
@@ -309,7 +309,7 @@
* <p>
* Finally, you can extend the low level {@link org.apache.lucene.search.similarities.Similarity Similarity} directly
* to implement a new retrieval model, or to use external scoring factors particular to your application. For example,
- * a custom Similarity can access per-document values via {@link org.apache.lucene.index.NumericDocValues} and
+ * a custom Similarity can access per-document values via {@link org.apache.lucene.index.NumericDocValues} and
* integrate them into the score.
* <p>
* See the {@link org.apache.lucene.search.similarities} package documentation for information
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index cbce18e..0cb86db 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -238,14 +238,27 @@ public class BM25Similarity extends Similarity {
}
@Override
- public float score(int doc, float freq) {
+ public float score(int doc, float freq) throws IOException {
// if there are no norms, we act as if b=0
- float norm = norms == null ? k1 : cache[(byte)norms.get(doc) & 0xFF];
+ float norm;
+ if (norms == null) {
+ norm = k1;
+ } else {
+ int normsDocID = norms.docID();
+ if (normsDocID < doc) {
+ normsDocID = norms.advance(doc);
+ }
+ if (normsDocID == doc) {
+ norm = cache[(byte)norms.longValue() & 0xFF];
+ } else {
+ norm = cache[0];
+ }
+ }
return weightValue * freq / (freq + norm);
}
@Override
- public Explanation explain(int doc, Explanation freq) {
+ public Explanation explain(int doc, Explanation freq) throws IOException {
return explainScore(doc, freq, stats, norms);
}
@@ -286,7 +299,7 @@ public class BM25Similarity extends Similarity {
}
- private Explanation explainTFNorm(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) {
+ private Explanation explainTFNorm(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) throws IOException {
List<Explanation> subs = new ArrayList<>();
subs.add(freq);
subs.add(Explanation.match(k1, "parameter k1"));
@@ -296,7 +309,13 @@ public class BM25Similarity extends Similarity {
(freq.getValue() * (k1 + 1)) / (freq.getValue() + k1),
"tfNorm, computed from:", subs);
} else {
- float doclen = decodeNormValue((byte)norms.get(doc));
+ byte norm;
+ if (norms.advance(doc) == doc) {
+ norm = (byte) norms.longValue();
+ } else {
+ norm = 0;
+ }
+ float doclen = decodeNormValue(norm);
subs.add(Explanation.match(b, "parameter b"));
subs.add(Explanation.match(stats.avgdl, "avgFieldLength"));
subs.add(Explanation.match(doclen, "fieldLength"));
@@ -306,7 +325,7 @@ public class BM25Similarity extends Similarity {
}
}
- private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) {
+ private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) throws IOException {
Explanation boostExpl = Explanation.match(stats.boost, "boost");
List<Explanation> subs = new ArrayList<>();
if (boostExpl.getValue() != 1.0f)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
index 68828fc..153fd7b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
@@ -75,7 +75,7 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public float score(int doc, float freq) {
+ public float score(int doc, float freq) throws IOException {
float sum = 0.0f;
for (SimScorer subScorer : subScorers) {
sum += subScorer.score(doc, freq);
@@ -84,7 +84,7 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public Explanation explain(int doc, Explanation freq) {
+ public Explanation explain(int doc, Explanation freq) throws IOException {
List<Explanation> subs = new ArrayList<>();
for (SimScorer subScorer : subScorers) {
subs.add(subScorer.explain(doc, freq));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
index 6a6e93b..7f0f27c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
@@ -161,7 +161,7 @@ public abstract class Similarity {
* @param freq sloppy term frequency
* @return document's score
*/
- public abstract float score(int doc, float freq);
+ public abstract float score(int doc, float freq) throws IOException;
/** Computes the amount of a sloppy phrase match, based on an edit distance. */
public abstract float computeSlopFactor(int distance);
@@ -175,7 +175,7 @@ public abstract class Similarity {
* @param freq Explanation of how the sloppy term frequency was computed
* @return document's score
*/
- public Explanation explain(int doc, Explanation freq) {
+ public Explanation explain(int doc, Explanation freq) throws IOException {
return Explanation.match(
score(doc, freq.getValue()),
"score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index c2c117d..ed837c4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -274,17 +274,31 @@ public abstract class SimilarityBase extends Similarity {
this.stats = stats;
this.norms = norms;
}
+
+ private float getNormValue(int doc) throws IOException {
+ if (norms == null) {
+ return 1F;
+ }
+ int normsDocID = norms.docID();
+ if (normsDocID < doc) {
+ normsDocID = norms.advance(doc);
+ }
+ if (normsDocID == doc) {
+ return decodeNormValue((byte) norms.longValue());
+ } else {
+ return decodeNormValue((byte) 0);
+ }
+ }
@Override
- public float score(int doc, float freq) {
+ public float score(int doc, float freq) throws IOException {
// We have to supply something in case norms are omitted
- return SimilarityBase.this.score(stats, freq,
- norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
+ return SimilarityBase.this.score(stats, freq, getNormValue(doc));
}
+
@Override
- public Explanation explain(int doc, Explanation freq) {
- return SimilarityBase.this.explain(stats, doc, freq,
- norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
+ public Explanation explain(int doc, Explanation freq) throws IOException {
+ return SimilarityBase.this.explain(stats, doc, freq, getNormValue(doc));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
index ff708b4..6cd87b5 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
@@ -592,10 +592,24 @@ public abstract class TFIDFSimilarity extends Similarity {
}
@Override
- public float score(int doc, float freq) {
+ public float score(int doc, float freq) throws IOException {
final float raw = tf(freq) * weightValue; // compute tf(f)*weight
-
- return norms == null ? raw : raw * decodeNormValue(norms.get(doc)); // normalize for field
+
+ if (norms == null) {
+ return raw;
+ } else {
+ long normValue;
+ int normsDocID = norms.docID();
+ if (normsDocID < doc) {
+ normsDocID = norms.advance(doc);
+ }
+ if (normsDocID == doc) {
+ normValue = norms.longValue();
+ } else {
+ normValue = 0;
+ }
+ return raw * decodeNormValue(normValue); // normalize for field
+ }
}
@Override
@@ -609,7 +623,7 @@ public abstract class TFIDFSimilarity extends Similarity {
}
@Override
- public Explanation explain(int doc, Explanation freq) {
+ public Explanation explain(int doc, Explanation freq) throws IOException {
return explainScore(doc, freq, stats, norms);
}
}
@@ -632,10 +646,17 @@ public abstract class TFIDFSimilarity extends Similarity {
}
}
- private Explanation explainField(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) {
+ private Explanation explainField(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) throws IOException {
Explanation tfExplanation = Explanation.match(tf(freq.getValue()), "tf(freq="+freq.getValue()+"), with freq of:", freq);
+ float norm;
+ if (norms != null && norms.advance(doc) == doc) {
+ norm = decodeNormValue(norms.longValue());
+ } else {
+ norm = 1f;
+ }
+
Explanation fieldNormExpl = Explanation.match(
- norms != null ? decodeNormValue(norms.get(doc)) : 1.0f,
+ norm,
"fieldNorm(doc=" + doc + ")");
return Explanation.match(
@@ -644,7 +665,7 @@ public abstract class TFIDFSimilarity extends Similarity {
tfExplanation, stats.idf, fieldNormExpl);
}
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) {
+ private Explanation explainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) throws IOException {
Explanation queryExpl = Explanation.match(stats.boost, "boost");
Explanation fieldExpl = explainField(doc, freq, stats, norms);
if (stats.boost == 1f) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/util/Bits.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Bits.java b/lucene/core/src/java/org/apache/lucene/util/Bits.java
index cfd3918..101122e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Bits.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Bits.java
@@ -43,7 +43,7 @@ public interface Bits {
public static class MatchAllBits implements Bits {
final int len;
- public MatchAllBits( int len ) {
+ public MatchAllBits(int len) {
this.len = len;
}
@@ -64,7 +64,7 @@ public interface Bits {
public static class MatchNoBits implements Bits {
final int len;
- public MatchNoBits( int len ) {
+ public MatchNoBits(int len) {
this.len = len;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/util/LongValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/LongValues.java b/lucene/core/src/java/org/apache/lucene/util/LongValues.java
index e4b8a35..aa3b7fa 100644
--- a/lucene/core/src/java/org/apache/lucene/util/LongValues.java
+++ b/lucene/core/src/java/org/apache/lucene/util/LongValues.java
@@ -17,15 +17,20 @@
package org.apache.lucene.util;
+import org.apache.lucene.index.LegacyNumericDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.util.packed.PackedInts;
/** Abstraction over an array of longs.
* This class extends NumericDocValues so that we don't need to add another
* level of abstraction every time we want eg. to use the {@link PackedInts}
- * utility classes to represent a {@link NumericDocValues} instance.
- * @lucene.internal */
-public abstract class LongValues extends NumericDocValues {
+ * utility classes to represent a {@link LegacyNumericDocValues} instance.
+ * @lucene.internal
+ *
+ * @deprecated Switch to {@link NumericDocValues} instead. */
+ @Deprecated
+// TODO: cutover to iterator once codecs have all cutover?
+public abstract class LongValues extends LegacyNumericDocValues {
/** An instance that returns the provided value. */
public static final LongValues IDENTITY = new LongValues() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java b/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
index e028c2c..054f3a3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
+++ b/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.LegacyNumericDocValues;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
@@ -449,7 +449,7 @@ public class PackedInts {
* A read-only random access array of positive integers.
* @lucene.internal
*/
- public static abstract class Reader extends NumericDocValues implements Accountable {
+ public static abstract class Reader extends LegacyNumericDocValues implements Accountable {
/**
* Bulk get: read at least one and at most <code>len</code> longs starting
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java
index 58d980e..f798148 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java
@@ -66,7 +66,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMFile;
import org.apache.lucene.store.RAMInputStream;
import org.apache.lucene.store.RAMOutputStream;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.LongValues;
@@ -206,19 +205,14 @@ public class TestLucene54DocValuesFormat extends BaseCompressingDocValuesFormatT
for (LeafReaderContext context : indexReader.leaves()) {
final LeafReader reader = context.reader();
final NumericDocValues numeric = DocValues.getNumeric(reader, "numeric");
- final Bits numericBits = DocValues.getDocsWithField(reader, "numeric");
final SortedDocValues sorted = DocValues.getSorted(reader, "sorted");
- final Bits sortedBits = DocValues.getDocsWithField(reader, "sorted");
final BinaryDocValues binary = DocValues.getBinary(reader, "binary");
- final Bits binaryBits = DocValues.getDocsWithField(reader, "binary");
final SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(reader, "sorted_numeric");
- final Bits sortedNumericBits = DocValues.getDocsWithField(reader, "sorted_numeric");
final SortedSetDocValues sortedSet = DocValues.getSortedSet(reader, "sorted_set");
- final Bits sortedSetBits = DocValues.getDocsWithField(reader, "sorted_set");
for (int i = 0; i < reader.maxDoc(); ++i) {
final Document doc = reader.document(i);
@@ -226,49 +220,43 @@ public class TestLucene54DocValuesFormat extends BaseCompressingDocValuesFormatT
final Long value = valueField == null ? null : valueField.numericValue().longValue();
if (value == null) {
- assertEquals(0, numeric.get(i));
- assertEquals(-1, sorted.getOrd(i));
- assertEquals(new BytesRef(), binary.get(i));
-
- assertFalse(numericBits.get(i));
- assertFalse(sortedBits.get(i));
- assertFalse(binaryBits.get(i));
+ assertTrue(numeric.docID() + " vs " + i, numeric.docID() < i);
} else {
- assertEquals(value.longValue(), numeric.get(i));
- assertTrue(sorted.getOrd(i) >= 0);
- assertEquals(new BytesRef(Long.toString(value)), sorted.lookupOrd(sorted.getOrd(i)));
- assertEquals(new BytesRef(Long.toString(value)), binary.get(i));
-
- assertTrue(numericBits.get(i));
- assertTrue(sortedBits.get(i));
- assertTrue(binaryBits.get(i));
+ assertEquals(i, numeric.nextDoc());
+ assertEquals(i, binary.nextDoc());
+ assertEquals(i, sorted.nextDoc());
+ assertEquals(value.longValue(), numeric.longValue());
+ assertTrue(sorted.ordValue() >= 0);
+ assertEquals(new BytesRef(Long.toString(value)), sorted.lookupOrd(sorted.ordValue()));
+ assertEquals(new BytesRef(Long.toString(value)), binary.binaryValue());
}
final IndexableField[] valuesFields = doc.getFields("values");
- final Set<Long> valueSet = new HashSet<>();
- for (IndexableField sf : valuesFields) {
- valueSet.add(sf.numericValue().longValue());
- }
+ if (valuesFields.length == 0) {
+ assertTrue(sortedNumeric.docID() + " vs " + i, sortedNumeric.docID() < i);
+ } else {
+ final Set<Long> valueSet = new HashSet<>();
+ for (IndexableField sf : valuesFields) {
+ valueSet.add(sf.numericValue().longValue());
+ }
- sortedNumeric.setDocument(i);
- assertEquals(valuesFields.length, sortedNumeric.count());
- for (int j = 0; j < sortedNumeric.count(); ++j) {
- assertTrue(valueSet.contains(sortedNumeric.valueAt(j)));
- }
- sortedSet.setDocument(i);
- int sortedSetCount = 0;
- while (true) {
- long ord = sortedSet.nextOrd();
- if (ord == SortedSetDocValues.NO_MORE_ORDS) {
- break;
+ assertEquals(i, sortedNumeric.nextDoc());
+ assertEquals(valuesFields.length, sortedNumeric.docValueCount());
+ for (int j = 0; j < sortedNumeric.docValueCount(); ++j) {
+ assertTrue(valueSet.contains(sortedNumeric.nextValue()));
+ }
+ assertEquals(i, sortedSet.nextDoc());
+ int sortedSetCount = 0;
+ while (true) {
+ long ord = sortedSet.nextOrd();
+ if (ord == SortedSetDocValues.NO_MORE_ORDS) {
+ break;
+ }
+ assertTrue(valueSet.contains(Long.parseLong(sortedSet.lookupOrd(ord).utf8ToString())));
+ sortedSetCount++;
}
- assertTrue(valueSet.contains(Long.parseLong(sortedSet.lookupOrd(ord).utf8ToString())));
- sortedSetCount++;
+ assertEquals(valueSet.size(), sortedSetCount);
}
- assertEquals(valueSet.size(), sortedSetCount);
-
- assertEquals(!valueSet.isEmpty(), sortedNumericBits.get(i));
- assertEquals(!valueSet.isEmpty(), sortedSetBits.get(i));
}
}
@@ -546,7 +534,7 @@ public class TestLucene54DocValuesFormat extends BaseCompressingDocValuesFormatT
RAMInputStream in = new RAMInputStream("", buffer);
BytesRefBuilder b = new BytesRefBuilder();
for (int i = 0; i < maxDoc; ++i) {
- values.setDocument(i);
+ assertEquals(i, values.nextDoc());
final int numValues = in.readVInt();
for (int j = 0; j < numValues; ++j) {
@@ -595,10 +583,10 @@ public class TestLucene54DocValuesFormat extends BaseCompressingDocValuesFormatT
assertNotNull(values);
RAMInputStream in = new RAMInputStream("", buffer);
for (int i = 0; i < maxDoc; ++i) {
- values.setDocument(i);
- assertEquals(2, values.count());
- assertEquals(in.readVLong(), values.valueAt(0));
- assertEquals(in.readVLong(), values.valueAt(1));
+ assertEquals(i, values.nextDoc());
+ assertEquals(2, values.docValueCount());
+ assertEquals(in.readVLong(), values.nextValue());
+ assertEquals(in.readVLong(), values.nextValue());
}
r.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
index 1ebfb69..5777dad 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
@@ -110,13 +110,17 @@ public class TestPerFieldDocValuesFormat extends BaseDocValuesFormatTestCase {
assertEquals(1, hits.totalHits);
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ int hitDocID = hits.scoreDocs[i].doc;
+ Document hitDoc = isearcher.doc(hitDocID);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
- assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ assertEquals(hitDocID, dv.advance(hitDocID));
+ assertEquals(5, dv.longValue());
+
BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
- final BytesRef term = dv2.get(hits.scoreDocs[i].doc);
+ assertEquals(hitDocID, dv2.advance(hitDocID));
+ final BytesRef term = dv2.binaryValue();
assertEquals(new BytesRef("hello world"), term);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java b/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
index 9f273df..4662480 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
@@ -91,7 +91,8 @@ public class Test2BBinaryDocValues extends LuceneTestCase {
bytes[1] = (byte)(expectedValue >> 16);
bytes[2] = (byte)(expectedValue >> 8);
bytes[3] = (byte) expectedValue;
- final BytesRef term = dv.get(i);
+ assertEquals(i, dv.nextDoc());
+ final BytesRef term = dv.binaryValue();
assertEquals(data, term);
expectedValue++;
}
@@ -148,7 +149,8 @@ public class Test2BBinaryDocValues extends LuceneTestCase {
LeafReader reader = context.reader();
BinaryDocValues dv = reader.getBinaryDocValues("dv");
for (int i = 0; i < reader.maxDoc(); i++) {
- final BytesRef term = dv.get(i);
+ assertEquals(i, dv.nextDoc());
+ final BytesRef term = dv.binaryValue();
input.reset(term.bytes, term.offset, term.length);
assertEquals(expectedValue % 65535, input.readVInt());
assertTrue(input.eof());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java b/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
index 2460073..046ac6f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
@@ -80,7 +80,8 @@ public class Test2BNumericDocValues extends LuceneTestCase {
LeafReader reader = context.reader();
NumericDocValues dv = reader.getNumericDocValues("dv");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(expectedValue, dv.get(i));
+ assertEquals(i, dv.nextDoc());
+ assertEquals(expectedValue, dv.longValue());
expectedValue++;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesFixedSorted.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesFixedSorted.java b/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesFixedSorted.java
index 56bf4b5..988e3f7 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesFixedSorted.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesFixedSorted.java
@@ -82,11 +82,12 @@ public class Test2BSortedDocValuesFixedSorted extends LuceneTestCase {
int expectedValue = 0;
for (LeafReaderContext context : r.leaves()) {
LeafReader reader = context.reader();
- BinaryDocValues dv = reader.getSortedDocValues("dv");
+ BinaryDocValues dv = DocValues.getBinary(reader, "dv");
for (int i = 0; i < reader.maxDoc(); i++) {
+ assertEquals(i, dv.nextDoc());
bytes[0] = (byte)(expectedValue >> 8);
bytes[1] = (byte) expectedValue;
- final BytesRef term = dv.get(i);
+ final BytesRef term = dv.binaryValue();
assertEquals(data, term);
expectedValue++;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesOrds.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesOrds.java b/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesOrds.java
index 70ed6bf..7448738 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesOrds.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValuesOrds.java
@@ -85,14 +85,15 @@ public class Test2BSortedDocValuesOrds extends LuceneTestCase {
for (LeafReaderContext context : r.leaves()) {
LeafReader reader = context.reader();
BytesRef scratch = new BytesRef();
- BinaryDocValues dv = reader.getSortedDocValues("dv");
+ BinaryDocValues dv = DocValues.getBinary(reader, "dv");
for (int i = 0; i < reader.maxDoc(); i++) {
+ assertEquals(i, dv.nextDoc());
bytes[0] = (byte) (counter >> 24);
bytes[1] = (byte) (counter >> 16);
bytes[2] = (byte) (counter >> 8);
bytes[3] = (byte) counter;
counter++;
- final BytesRef term = dv.get(i);
+ final BytesRef term = dv.binaryValue();
assertEquals(data, term);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
index c6cf1c3..f552ae9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
@@ -46,12 +46,15 @@ import org.junit.Test;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
public class TestBinaryDocValuesUpdates extends LuceneTestCase {
- static long getValue(BinaryDocValues bdv, int idx) {
- BytesRef term = bdv.get(idx);
- idx = term.offset;
+ static long getValue(BinaryDocValues bdv) {
+ BytesRef term = bdv.binaryValue();
+ int idx = term.offset;
+ assert term.length > 0;
byte b = term.bytes[idx++];
long value = b & 0x7FL;
for (int shift = 7; (b & 0x80L) != 0; shift += 7) {
@@ -129,8 +132,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
assertEquals(1, reader.leaves().size());
LeafReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("val");
- assertEquals(2, getValue(bdv, 0));
- assertEquals(2, getValue(bdv, 1));
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(2, getValue(bdv));
+ assertEquals(1, bdv.nextDoc());
+ assertEquals(2, getValue(bdv));
reader.close();
dir.close();
@@ -173,8 +178,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = r.getBinaryDocValues("val");
assertNotNull(bdv);
for (int i = 0; i < r.maxDoc(); i++) {
+ assertEquals(i, bdv.nextDoc());
long expected = expectedValues[i + context.docBase];
- long actual = getValue(bdv, i);
+ long actual = getValue(bdv);
assertEquals(expected, actual);
}
}
@@ -212,8 +218,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv1 = reader1.leaves().get(0).reader().getBinaryDocValues("val");
BinaryDocValues bdv2 = reader2.leaves().get(0).reader().getBinaryDocValues("val");
- assertEquals(1, getValue(bdv1, 0));
- assertEquals(10, getValue(bdv2, 0));
+ assertEquals(0, bdv1.nextDoc());
+ assertEquals(1, getValue(bdv1));
+ assertEquals(0, bdv2.nextDoc());
+ assertEquals(10, getValue(bdv2));
writer.close();
IOUtils.close(reader1, reader2, dir);
@@ -260,7 +268,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17};
BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "val");
for (int i = 0; i < expectedValues.length; i++) {
- assertEquals(expectedValues[i], getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(expectedValues[i], getValue(bdv));
}
reader.close();
@@ -295,7 +304,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
LeafReader r = reader.leaves().get(0).reader();
assertFalse(r.getLiveDocs().get(0));
- assertEquals(17, getValue(r.getBinaryDocValues("val"), 1));
+ BinaryDocValues bdv = r.getBinaryDocValues("val");
+ assertEquals(1, bdv.advance(1));
+ assertEquals(17, getValue(bdv));
reader.close();
dir.close();
@@ -330,14 +341,18 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
SortedDocValues sdv = r.getSortedDocValues("sdv");
SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(i, ndv.get(i));
- assertEquals(17, getValue(bdv, i));
- BytesRef term = sdv.get(i);
+ assertEquals(i, ndv.nextDoc());
+ assertEquals(i, ndv.longValue());
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(17, getValue(bdv));
+ assertEquals(i, sdv.nextDoc());
+ BytesRef term = sdv.binaryValue();
assertEquals(new BytesRef(Integer.toString(i)), term);
- ssdv.setDocument(i);
+ assertEquals(i, ssdv.nextDoc());
long ord = ssdv.nextOrd();
term = ssdv.lookupOrd(ord);
assertEquals(i, Integer.parseInt(term.utf8ToString()));
+ // For the i=0 case, we added the same value twice, which was dedup'd by IndexWriter so it has only one value:
if (i != 0) {
ord = ssdv.nextOrd();
term = ssdv.lookupOrd(ord);
@@ -375,8 +390,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv1 = r.getBinaryDocValues("bdv1");
BinaryDocValues bdv2 = r.getBinaryDocValues("bdv2");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(17, getValue(bdv1, i));
- assertEquals(i, getValue(bdv2, i));
+ assertEquals(i, bdv1.nextDoc());
+ assertEquals(17, getValue(bdv1));
+ assertEquals(i, bdv2.nextDoc());
+ assertEquals(i, getValue(bdv2));
}
reader.close();
@@ -406,7 +423,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
LeafReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(17, getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(17, getValue(bdv));
}
reader.close();
@@ -468,8 +486,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
SortedDocValues sdv = MultiDocValues.getSortedValues(reader, "sorted");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(17, getValue(bdv, i));
- BytesRef term = sdv.get(i);
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(17, getValue(bdv));
+ assertEquals(i, sdv.nextDoc());
+ BytesRef term = sdv.binaryValue();
assertEquals(new BytesRef("value"), term);
}
@@ -496,7 +516,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(3, getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(3, getValue(bdv));
}
reader.close();
dir.close();
@@ -507,7 +528,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
Random random = random();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
IndexWriter writer = new IndexWriter(dir, conf);
-
+
int docid = 0;
int numRounds = atLeast(10);
for (int rnd = 0; rnd < numRounds; rnd++) {
@@ -564,7 +585,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
assertNotNull(bdv);
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(value, getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(value, getValue(bdv));
}
reader.close();
}
@@ -594,7 +616,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertEquals(3, getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(3, getValue(bdv));
}
reader.close();
dir.close();
@@ -672,14 +695,12 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
for (int field = 0; field < fieldValues.length; field++) {
String f = "f" + field;
BinaryDocValues bdv = r.getBinaryDocValues(f);
- Bits docsWithField = r.getDocsWithField(f);
assertNotNull(bdv);
int maxDoc = r.maxDoc();
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
-// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
- assertTrue(docsWithField.get(doc));
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc));
+ assertEquals(doc, bdv.advance(doc));
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv));
}
}
}
@@ -732,13 +753,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
for (LeafReaderContext context : reader.leaves()) {
LeafReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
- Bits docsWithField = r.getDocsWithField("bdv");
- assertNotNull(docsWithField);
- assertTrue(docsWithField.get(0));
- assertEquals(5L, getValue(bdv, 0));
- assertFalse(docsWithField.get(1));
- BytesRef term = bdv.get(1);
- assertEquals(0, term.length);
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(5L, getValue(bdv));
+ assertEquals(NO_MORE_DOCS, bdv.nextDoc());
}
reader.close();
@@ -778,7 +795,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
LeafReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(5L, getValue(bdv, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(5L, getValue(bdv));
}
}
reader.close();
@@ -803,7 +821,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(dir);
BinaryDocValues bdv = r.leaves().get(0).reader().getBinaryDocValues("f");
- assertEquals(17, getValue(bdv, 0));
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(17, getValue(bdv));
r.close();
dir.close();
@@ -920,14 +939,12 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
for (int i = 0; i < numFields; i++) {
BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
BinaryDocValues control = r.getBinaryDocValues("cf" + i);
- Bits docsWithBdv = r.getDocsWithField("f" + i);
- Bits docsWithControl = r.getDocsWithField("cf" + i);
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
- assertTrue(docsWithBdv.get(j));
- assertTrue(docsWithControl.get(j));
- assertEquals(getValue(control, j), getValue(bdv, j) * 2);
+ assertEquals(j, bdv.advance(j));
+ assertEquals(j, control.advance(j));
+ assertEquals(getValue(control), getValue(bdv) * 2);
}
}
}
@@ -965,7 +982,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues fbdv = r.getBinaryDocValues("f");
BinaryDocValues cfbdv = r.getBinaryDocValues("cf");
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals(getValue(cfbdv, j), getValue(fbdv, j) * 2);
+ assertEquals(j, fbdv.nextDoc());
+ assertEquals(j, cfbdv.nextDoc());
+ assertEquals(getValue(cfbdv), getValue(fbdv) * 2);
}
}
reader.close();
@@ -1013,10 +1032,14 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
BinaryDocValues f1 = MultiDocValues.getBinaryValues(reader, "f1");
BinaryDocValues f2 = MultiDocValues.getBinaryValues(reader, "f2");
- assertEquals(12L, getValue(f1, 0));
- assertEquals(13L, getValue(f2, 0));
- assertEquals(17L, getValue(f1, 1));
- assertEquals(2L, getValue(f2, 1));
+ assertEquals(0, f1.nextDoc());
+ assertEquals(0, f2.nextDoc());
+ assertEquals(12L, getValue(f1));
+ assertEquals(13L, getValue(f2));
+ assertEquals(1, f1.nextDoc());
+ assertEquals(1, f2.nextDoc());
+ assertEquals(17L, getValue(f1));
+ assertEquals(2L, getValue(f2));
reader.close();
dir.close();
}
@@ -1070,7 +1093,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BinaryDocValues control = r.getBinaryDocValues("control");
for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(getValue(bdv, i)*2, getValue(control, i));
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(i, control.nextDoc());
+ assertEquals(getValue(bdv)*2, getValue(control));
}
}
reader.close();
@@ -1167,7 +1192,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues f = r.getBinaryDocValues("f" + i);
BinaryDocValues cf = r.getBinaryDocValues("cf" + i);
for (int j = 0; j < r.maxDoc(); j++) {
- assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, getValue(cf, j), getValue(f, j) * 2);
+ assertEquals(j, f.nextDoc());
+ assertEquals(j, cf.nextDoc());
+ assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, getValue(cf), getValue(f) * 2);
}
}
}
@@ -1195,8 +1222,12 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.close();
DirectoryReader reader = DirectoryReader.open(dir);
- assertEquals(4, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
- assertEquals(3, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f2"), 0));
+ BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(4, getValue(bdv));
+ bdv = reader.leaves().get(0).reader().getBinaryDocValues("f2");
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(3, getValue(bdv));
reader.close();
dir.close();
@@ -1220,7 +1251,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
- assertEquals(2L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
+ BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(2L, getValue(bdv));
reader.close();
dir.close();
@@ -1242,7 +1275,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
- assertEquals(1L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
+ BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
+ assertEquals(0, bdv.nextDoc());
+ assertEquals(1L, getValue(bdv));
reader.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
index ba3414b..65310cf 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
@@ -72,7 +72,8 @@ public class TestCustomNorms extends LuceneTestCase {
for (int i = 0; i < open.maxDoc(); i++) {
Document document = open.document(i);
float expected = Float.parseFloat(document.get(floatTestField));
- assertEquals(expected, Float.intBitsToFloat((int)norms.get(i)), 0.0f);
+ assertEquals(i, norms.nextDoc());
+ assertEquals(expected, Float.intBitsToFloat((int)norms.longValue()), 0.0f);
}
open.close();
dir.close();