You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/09/21 13:42:13 UTC
[09/16] lucene-solr:master: LUCENE-7407: switch doc values usage to
an iterator API, based on DocIdSetIterator, instead of random acces,
freeing codecs for future improvements
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index cf8372d..af3d391 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -513,7 +513,6 @@ public class TestPointQueries extends LuceneTestCase {
private void _run() throws Exception {
startingGun.await();
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
for (int iter=0;iter<iters && failed.get() == false;iter++) {
Long lower = randomValue();
Long upper = randomValue();
@@ -578,8 +577,11 @@ public class TestPointQueries extends LuceneTestCase {
System.out.println(Thread.currentThread().getName() + ": hitCount: " + hits.cardinality());
}
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
for(int docID=0;docID<r.maxDoc();docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
boolean expected = missing.get(id) == false && deleted.get(id) == false && values[id] >= lower && values[id] <= upper;
if (hits.get(docID) != expected) {
// We do exact quantized comparison so the bbox query should never disagree:
@@ -768,8 +770,6 @@ public class TestPointQueries extends LuceneTestCase {
private void _run() throws Exception {
startingGun.await();
- NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
-
for (int iter=0;iter<iters && failed.get() == false;iter++) {
byte[][] lower = new byte[numDims][];
@@ -837,9 +837,12 @@ public class TestPointQueries extends LuceneTestCase {
}
}
+ NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
int failCount = 0;
for(int docID=0;docID<r.maxDoc();docID++) {
- int id = (int) docIDToID.get(docID);
+ assertEquals(docID, docIDToID.nextDoc());
+ int id = (int) docIDToID.longValue();
if (hits.get(docID) != expected.get(id)) {
System.out.println("FAIL: iter=" + iter + " id=" + id + " docID=" + docID + " expected=" + expected.get(id) + " but got " + hits.get(docID) + " deleted?=" + deleted.get(id) + " missing?=" + missing.get(id));
for(int dim=0;dim<numDims;dim++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index 83144b1..de96755 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -77,7 +77,9 @@ public class TestSimilarityProvider extends LuceneTestCase {
NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar");
for (int i = 0; i < reader.maxDoc(); i++) {
- assertFalse(fooNorms.get(i) == barNorms.get(i));
+ assertEquals(i, fooNorms.nextDoc());
+ assertEquals(i, barNorms.nextDoc());
+ assertFalse(fooNorms.longValue() == barNorms.longValue());
}
// sanity check of searching
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
index 2bccaff..5c06a74 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
@@ -239,10 +239,11 @@ public class TestSortRandom extends LuceneTestCase {
assertNotNull(idSource);
final FixedBitSet bits = new FixedBitSet(maxDoc);
for(int docID=0;docID<maxDoc;docID++) {
+ assertEquals(docID, idSource.nextDoc());
if (random.nextFloat() <= density) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
- matchValues.add(docValues.get((int) idSource.get(docID)));
+ matchValues.add(docValues.get((int) idSource.longValue()));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
index 774a2ce..4287125 100644
--- a/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
+++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
@@ -19,7 +19,7 @@ package org.apache.lucene.util.packed;
import java.util.Random;
-import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.LegacyNumericDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -46,7 +46,7 @@ public class TestDirectPacked extends LuceneTestCase {
writer.finish();
output.close();
IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
- NumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
+ LegacyNumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
assertEquals(1, reader.get(0));
assertEquals(0, reader.get(1));
assertEquals(2, reader.get(2));
@@ -110,7 +110,7 @@ public class TestDirectPacked extends LuceneTestCase {
writer.finish();
output.close();
IndexInput input = directory.openInput(name, IOContext.DEFAULT);
- NumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsRequired, offset);
+ LegacyNumericDocValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsRequired, offset);
for (int j = 0; j < original.length; j++) {
assertEquals("bpv=" + bpv, original[j], reader.get(j));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java b/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
index e64249e..1c11cb2 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
@@ -36,7 +36,7 @@ import org.apache.lucene.search.SortField;
* SimpleBindings bindings = new SimpleBindings();
* // document's text relevance score
* bindings.add(new SortField("_score", SortField.Type.SCORE));
- * // integer NumericDocValues field (or from FieldCache)
+ * // integer NumericDocValues field
* bindings.add(new SortField("popularity", SortField.Type.INT));
* // another expression
* bindings.add("recency", myRecencyExpression);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
index c78f6a97..13174bc 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
@@ -154,7 +154,7 @@ public final class JavascriptCompiler {
* use the FunctionValues class.
*/
@SuppressWarnings({"unused", "null"})
- private static void unusedTestCompile() {
+ private static void unusedTestCompile() throws IOException {
FunctionValues f = null;
double ret = f.doubleVal(2);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
index 3129d8c..eeb3c9c 100644
--- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
+++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
@@ -136,6 +136,7 @@ public class TestExpressionValueSource extends LuceneTestCase {
assertEquals(DocIdSetIterator.NO_MORE_DOCS, iter.nextDoc());
// just the first doc
+ values = vs.getValues(new HashMap<String,Object>(), leaf);
scorer = values.getRangeScorer(leaf, "4", "40", false, false);
iter = scorer.iterator();
assertEquals(-1, scorer.docID());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
index 936c962..7bbe94a 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
@@ -113,8 +113,8 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
if (map == null) {
// uncached, or not a multi dv
SortedSetDocValues dv = MultiDocValues.getSortedSetValues(origReader, field);
- if (dv instanceof MultiSortedSetDocValues) {
- map = ((MultiSortedSetDocValues)dv).mapping;
+ if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
+ map = ((MultiDocValues.MultiSortedSetDocValues)dv).mapping;
if (map.owner == origReader.getCoreCacheKey()) {
cachedOrdMaps.put(field, map);
}
@@ -127,6 +127,7 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
int size = origReader.leaves().size();
final SortedSetDocValues[] values = new SortedSetDocValues[size];
final int[] starts = new int[size+1];
+ long cost = 0;
for (int i = 0; i < size; i++) {
LeafReaderContext context = origReader.leaves().get(i);
final LeafReader reader = context.reader();
@@ -140,9 +141,10 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
}
values[i] = v;
starts[i] = context.docBase;
+ cost += v.cost();
}
starts[size] = origReader.maxDoc();
- return new MultiSortedSetDocValues(values, starts, map);
+ return new MultiSortedSetDocValues(values, starts, map, cost);
}
/** Returns mapping from prefix to {@link OrdRange}. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
index a571328..38fb174 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
@@ -25,16 +25,16 @@ import java.util.Map;
import org.apache.lucene.facet.FacetResult;
import org.apache.lucene.facet.Facets;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.TopOrdAndIntQueue;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState.OrdRange;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -151,7 +151,7 @@ public class SortedSetDocValuesFacetCounts extends Facets {
// TODO: is this right? really, we need a way to
// verify that this ordinalMap "matches" the leaves in
// matchingDocs...
- if (dv instanceof MultiSortedSetDocValues && matchingDocs.size() > 1) {
+ if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
} else {
ordinalMap = null;
@@ -199,12 +199,16 @@ public class SortedSetDocValuesFacetCounts extends Facets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- //System.out.println(" segOrd=" + segOrd + " ord=" + term + " globalOrd=" + ordinalMap.getGlobalOrd(segOrd, term));
- counts[(int) ordMap.get(term)]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ //System.out.println(" segOrd=" + segOrd + " ord=" + term + " globalOrd=" + ordinalMap.getGlobalOrd(segOrd, term));
+ counts[(int) ordMap.get(term)]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
} else {
@@ -215,12 +219,16 @@ public class SortedSetDocValuesFacetCounts extends Facets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- //System.out.println(" ord=" + term);
- segCounts[term]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ //System.out.println(" ord=" + term);
+ segCounts[term]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
@@ -238,11 +246,15 @@ public class SortedSetDocValuesFacetCounts extends Facets {
// just aggregate directly into counts:
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- segValues.setDocument(doc);
- int term = (int) segValues.nextOrd();
- while (term != SortedSetDocValues.NO_MORE_ORDS) {
- counts[term]++;
- term = (int) segValues.nextOrd();
+ if (doc > segValues.docID()) {
+ segValues.advance(doc);
+ }
+ if (doc == segValues.docID()) {
+ int term = (int) segValues.nextOrd();
+ while (term != SortedSetDocValues.NO_MORE_ORDS) {
+ counts[term]++;
+ term = (int) segValues.nextOrd();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
index 6afe5c7..fd48849 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
@@ -19,9 +19,9 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -50,9 +50,24 @@ public class DocValuesOrdinalsReader extends OrdinalsReader {
final BinaryDocValues values = values0;
return new OrdinalsSegmentReader() {
+
+ private int lastDocID;
+
@Override
public void get(int docID, IntsRef ordinals) throws IOException {
- final BytesRef bytes = values.get(docID);
+ if (docID < lastDocID) {
+ throw new AssertionError("docs out of order: lastDocID=" + lastDocID + " vs docID=" + docID);
+ }
+ lastDocID = docID;
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ final BytesRef bytes;
+ if (values.docID() == docID) {
+ bytes = values.binaryValue();
+ } else {
+ bytes = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
decode(bytes, ordinals);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
index 0ed8f22..7ad5430 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -59,20 +59,25 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int ord = 0;
- int offset = bytesRef.offset;
- int prev = 0;
- while (offset < end) {
- byte b = bytes[offset++];
- if (b >= 0) {
- prev = ord = ((ord << 7) | b) + prev;
- ++values[ord];
- ord = 0;
- } else {
- ord = (ord << 7) | (b & 0x7F);
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int ord = 0;
+ int offset = bytesRef.offset;
+ int prev = 0;
+ while (offset < end) {
+ byte b = bytes[offset++];
+ if (b >= 0) {
+ prev = ord = ((ord << 7) | b) + prev;
+ ++values[ord];
+ ord = 0;
+ } else {
+ ord = (ord << 7) | (b & 0x7F);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
index f116cd9..cb798af 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/OrdinalMappingLeafReader.java
@@ -20,13 +20,14 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.FacetsConfig.DimConfig;
+import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.taxonomy.OrdinalsReader.OrdinalsSegmentReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FilterBinaryDocValues;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -75,25 +76,26 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
}
- private class OrdinalMappingBinaryDocValues extends BinaryDocValues {
+ private class OrdinalMappingBinaryDocValues extends FilterBinaryDocValues {
private final IntsRef ordinals = new IntsRef(32);
private final OrdinalsSegmentReader ordsReader;
- OrdinalMappingBinaryDocValues(OrdinalsSegmentReader ordsReader) throws IOException {
+ OrdinalMappingBinaryDocValues(OrdinalsSegmentReader ordsReader, BinaryDocValues in) throws IOException {
+ super(in);
this.ordsReader = ordsReader;
}
@SuppressWarnings("synthetic-access")
@Override
- public BytesRef get(int docID) {
+ public BytesRef binaryValue() {
try {
// NOTE: this isn't quite koscher, because in general
// multiple threads can call BinaryDV.get which would
// then conflict on the single ordinals instance, but
// because this impl is only used for merging, we know
// only 1 thread calls us:
- ordsReader.get(docID, ordinals);
+ ordsReader.get(docID(), ordinals);
// map the ordinals
for (int i = 0; i < ordinals.length; i++) {
@@ -102,7 +104,7 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
return encode(ordinals);
} catch (IOException e) {
- throw new RuntimeException("error reading category ordinals for doc " + docID, e);
+ throw new RuntimeException("error reading category ordinals for doc " + docID(), e);
}
}
}
@@ -150,7 +152,7 @@ public class OrdinalMappingLeafReader extends FilterLeafReader {
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
if (facetFields.contains(field)) {
final OrdinalsReader ordsReader = getOrdinalsReader(field);
- return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext()));
+ return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext()), in.getBinaryDocValues(field));
} else {
return in.getBinaryDocValues(field);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
index 7dd31d3..ce82e27 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
index a76e750..5a71aa2 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -61,22 +61,27 @@ public class TaxonomyFacetSumFloatAssociations extends FloatTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int offset = bytesRef.offset;
- while (offset < end) {
- int ord = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- int value = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- values[ord] += Float.intBitsToFloat(value);
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int offset = bytesRef.offset;
+ while (offset < end) {
+ int ord = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ int value = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ values[ord] += Float.intBitsToFloat(value);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
index 64001ad..6913d0d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
@@ -19,8 +19,8 @@ package org.apache.lucene.facet.taxonomy;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
@@ -61,22 +61,27 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
- final BytesRef bytesRef = dv.get(doc);
- byte[] bytes = bytesRef.bytes;
- int end = bytesRef.offset + bytesRef.length;
- int offset = bytesRef.offset;
- while (offset < end) {
- int ord = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- int value = ((bytes[offset]&0xFF) << 24) |
- ((bytes[offset+1]&0xFF) << 16) |
- ((bytes[offset+2]&0xFF) << 8) |
- (bytes[offset+3]&0xFF);
- offset += 4;
- values[ord] += value;
+ if (dv.docID() < doc) {
+ dv.advance(doc);
+ }
+ if (dv.docID() == doc) {
+ final BytesRef bytesRef = dv.binaryValue();
+ byte[] bytes = bytesRef.bytes;
+ int end = bytesRef.offset + bytesRef.length;
+ int offset = bytesRef.offset;
+ while (offset < end) {
+ int ord = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ int value = ((bytes[offset]&0xFF) << 24) |
+ ((bytes[offset+1]&0xFF) << 16) |
+ ((bytes[offset+2]&0xFF) << 8) |
+ (bytes[offset+3]&0xFF);
+ offset += 4;
+ values[ord] += value;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java b/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
index ea29c68..1a7633b 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/TestMultipleIndexFields.java
@@ -296,4 +296,4 @@ public class TestMultipleIndexFields extends FacetTestCase {
iw.addDocument(config.build(tw, doc));
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
index 0978b50..db3c59e 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestOrdinalMappingLeafReader.java
@@ -28,8 +28,8 @@ import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
@@ -106,7 +106,9 @@ public class TestOrdinalMappingLeafReader extends FacetTestCase {
BinaryDocValues bdv = MultiDocValues.getBinaryValues(indexReader, "bdv");
BinaryDocValues cbdv = MultiDocValues.getBinaryValues(indexReader, "cbdv");
for (int i = 0; i < indexReader.maxDoc(); i++) {
- assertEquals(Integer.parseInt(cbdv.get(i).utf8ToString()), Integer.parseInt(bdv.get(i).utf8ToString())*2);
+ assertEquals(i, bdv.nextDoc());
+ assertEquals(i, cbdv.nextDoc());
+ assertEquals(Integer.parseInt(cbdv.binaryValue().utf8ToString()), Integer.parseInt(bdv.binaryValue().utf8ToString())*2);
}
IOUtils.close(indexReader, taxoReader);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
index 4c386b6..312a3e2 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java
@@ -338,7 +338,7 @@ abstract public class AbstractFirstPassGroupingCollector<GROUP_VALUE_TYPE> exten
* @param doc The specified doc
* @return the group value for the specified doc
*/
- protected abstract GROUP_VALUE_TYPE getDocGroupValue(int doc);
+ protected abstract GROUP_VALUE_TYPE getDocGroupValue(int doc) throws IOException;
/**
* Returns a copy of the specified group value by creating a new instance and copying the value from the specified
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
index 36579a6..0237e21 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
@@ -61,7 +61,7 @@ public class FunctionFirstPassGroupingCollector extends AbstractFirstPassGroupin
}
@Override
- protected MutableValue getDocGroupValue(int doc) {
+ protected MutableValue getDocGroupValue(int doc) throws IOException {
filler.fillValue(doc);
return mval;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
index ae01e33..14636ec 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
@@ -16,8 +16,15 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
@@ -29,17 +36,10 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* A base implementation of {@link org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector} for retrieving the most relevant groups when grouping
* on a string based group field. More specifically this all concrete implementations of this base implementation
- * use {@link org.apache.lucene.index.SortedDocValues}.
+ * use {@link SortedDocValues}.
*
* @lucene.experimental
*/
@@ -131,15 +131,30 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ protected int getOrdForDoc(int doc) throws IOException {
+ int curDocID = groupIndex.docID();
+ if (curDocID < doc) {
+ curDocID = groupIndex.advance(doc);
+ }
+
+ if (curDocID == doc) {
+ return groupIndex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- final int ord = groupIndex.getOrd(doc);
+ int ord = getOrdForDoc(doc);
+
BytesRef groupValue;
if (ord == -1) {
groupValue = null;
} else {
groupValue = groupIndex.lookupOrd(ord);
}
+
GroupHead groupHead = groups.get(groupValue);
if (groupHead == null) {
groupValue = groupValue == null ? null : BytesRef.deepCopyOf(groupValue);
@@ -263,9 +278,22 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
this.scorer = scorer;
}
+ private int getOrdForDoc(int doc) throws IOException {
+ int curDocID = groupIndex.docID();
+ if (curDocID < doc) {
+ curDocID = groupIndex.advance(doc);
+ }
+
+ if (curDocID == doc) {
+ return groupIndex.ordValue();
+ } else {
+ return -1;
+ }
+ }
+
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ int key = getOrdForDoc(doc);
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
@@ -328,6 +356,15 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ void setDoc(int docID) throws IOException {
+ for (int i = 0; i < sortsIndex.length; i++) {
+ SortedDocValues values = sortsIndex[i];
+ if (values != null && docID > values.docID()) {
+ values.advance(docID);
+ }
+ }
+ }
+
class GroupHead extends AbstractAllGroupHeadsCollector.GroupHead<BytesRef> {
BytesRefBuilder[] sortValues;
@@ -339,14 +376,19 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
sortValues = new BytesRefBuilder[sortsIndex.length];
sortOrds = new int[sortsIndex.length];
scores = new float[sortsIndex.length];
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
if (fields[i].getType() == SortField.Type.SCORE) {
scores[i] = scorer.score();
} else {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
+ if (doc == sortsIndex[i].docID()) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ } else {
+ sortOrds[i] = -1;
+ }
sortValues[i] = new BytesRefBuilder();
if (sortOrds[i] != -1) {
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
}
}
}
@@ -363,24 +405,44 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
return 0;
} else {
+ if (sortsIndex[compIDX].docID() < doc) {
+ sortsIndex[compIDX].advance(doc);
+ }
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- final BytesRef term = sortsIndex[compIDX].get(doc);
+ final BytesRef term;
+ if (sortsIndex[compIDX].docID() == doc) {
+ term = sortsIndex[compIDX].binaryValue();
+ } else {
+ term = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
return sortValues[compIDX].get().compareTo(term);
} else {
- return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
+ int ord;
+ if (sortsIndex[compIDX].docID() == doc) {
+ ord = sortsIndex[compIDX].ordValue();
+ } else {
+ ord = -1;
+ }
+ return sortOrds[compIDX] - ord;
}
}
}
@Override
public void updateDocHead(int doc) throws IOException {
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
if (fields[i].getType() == SortField.Type.SCORE) {
scores[i] = scorer.score();
} else {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortsIndex[i].docID() == doc) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ } else {
+ sortOrds[i] = -1;
+ sortValues[i].clear();
+ }
}
}
this.doc = doc + readerContext.docBase;
@@ -429,7 +491,17 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ if (doc > groupIndex.docID()) {
+ groupIndex.advance(doc);
+ }
+
+ int key;
+ if (doc == groupIndex.docID()) {
+ key = groupIndex.ordValue();
+ } else {
+ key = -1;
+ }
+
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
@@ -485,38 +557,74 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
}
}
+ void setDoc(int docID) throws IOException {
+ for (int i = 0; i < sortsIndex.length; i++) {
+ SortedDocValues values = sortsIndex[i];
+ if (docID > values.docID()) {
+ values.advance(docID);
+ }
+ }
+ }
+
class GroupHead extends AbstractAllGroupHeadsCollector.GroupHead<BytesRef> {
BytesRefBuilder[] sortValues;
int[] sortOrds;
- GroupHead(int doc, BytesRef groupValue) {
+ GroupHead(int doc, BytesRef groupValue) throws IOException {
super(groupValue, doc + readerContext.docBase);
sortValues = new BytesRefBuilder[sortsIndex.length];
sortOrds = new int[sortsIndex.length];
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
+ if (doc == sortsIndex[i].docID()) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ } else {
+ sortOrds[i] = -1;
+ }
sortValues[i] = new BytesRefBuilder();
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortOrds[i] != -1) {
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ }
}
}
@Override
public int compare(int compIDX, int doc) throws IOException {
+ if (sortsIndex[compIDX].docID() < doc) {
+ sortsIndex[compIDX].advance(doc);
+ }
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- final BytesRef term = sortsIndex[compIDX].get(doc);
+ final BytesRef term;
+ if (sortsIndex[compIDX].docID() == doc) {
+ term = sortsIndex[compIDX].binaryValue();
+ } else {
+ term = new BytesRef(BytesRef.EMPTY_BYTES);
+ }
return sortValues[compIDX].get().compareTo(term);
} else {
- return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
+ int ord;
+ if (sortsIndex[compIDX].docID() == doc) {
+ ord = sortsIndex[compIDX].ordValue();
+ } else {
+ ord = -1;
+ }
+ return sortOrds[compIDX] - ord;
}
}
@Override
public void updateDocHead(int doc) throws IOException {
+ setDoc(doc);
for (int i = 0; i < sortsIndex.length; i++) {
- sortOrds[i] = sortsIndex[i].getOrd(doc);
- sortValues[i].copyBytes(sortsIndex[i].get(doc));
+ if (sortsIndex[i].docID() == doc) {
+ sortOrds[i] = sortsIndex[i].ordValue();
+ sortValues[i].copyBytes(sortsIndex[i].binaryValue());
+ } else {
+ sortOrds[i] = -1;
+ sortValues[i].clear();
+ }
}
this.doc = doc + readerContext.docBase;
}
@@ -566,7 +674,17 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
- int key = groupIndex.getOrd(doc);
+ if (doc > groupIndex.docID()) {
+ groupIndex.advance(doc);
+ }
+
+ int key;
+ if (doc == groupIndex.docID()) {
+ key = groupIndex.ordValue();
+ } else {
+ key = -1;
+ }
+
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
index f79964d..0009c36 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
@@ -16,18 +16,18 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
/**
* A collector that collects all groups that match the
* query. Only the group value is collected, and the order
@@ -81,7 +81,15 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
@Override
public void collect(int doc) throws IOException {
- int key = index.getOrd(doc);
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+ int key;
+ if (doc == index.docID()) {
+ key = index.ordValue();
+ } else {
+ key = -1;
+ }
if (!ordSet.exists(key)) {
ordSet.put(key);
final BytesRef term;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
index ab492d0..7980c48 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
@@ -16,20 +16,20 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.grouping.AbstractDistinctValuesCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
/**
* A term based implementation of {@link org.apache.lucene.search.grouping.AbstractDistinctValuesCollector} that relies
* on {@link SortedDocValues} to count the distinct values per group.
@@ -67,13 +67,32 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
@Override
public void collect(int doc) throws IOException {
- int slot = ordSet.find(groupFieldTermIndex.getOrd(doc));
+ if (doc > groupFieldTermIndex.docID()) {
+ groupFieldTermIndex.advance(doc);
+ }
+ int ord;
+ if (doc == groupFieldTermIndex.docID()) {
+ ord = groupFieldTermIndex.ordValue();
+ } else {
+ ord = -1;
+ }
+ int slot = ordSet.find(ord);
if (slot < 0) {
return;
}
GroupCount gc = groupCounts[slot];
- int countOrd = countFieldTermIndex.getOrd(doc);
+ if (doc > countFieldTermIndex.docID()) {
+ countFieldTermIndex.advance(doc);
+ }
+
+ int countOrd;
+ if (doc == countFieldTermIndex.docID()) {
+ countOrd = countFieldTermIndex.ordValue();
+ } else {
+ countOrd = -1;
+ }
+
if (doesNotContainOrd(countOrd, gc.ords)) {
if (countOrd == -1) {
gc.uniqueValues.add(null);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
index 4c50254..063521f 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
@@ -18,8 +18,8 @@ package org.apache.lucene.search.grouping.term;
import java.io.IOException;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector;
@@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector} that groups based on
- * field values and more specifically uses {@link org.apache.lucene.index.SortedDocValues}
+ * field values and more specifically uses {@link SortedDocValues}
* to collect groups.
*
* @lucene.experimental
@@ -61,12 +61,14 @@ public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCol
}
@Override
- protected BytesRef getDocGroupValue(int doc) {
- final int ord = index.getOrd(doc);
- if (ord == -1) {
- return null;
+ protected BytesRef getDocGroupValue(int doc) throws IOException {
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+ if (doc == index.docID()) {
+ return index.binaryValue();
} else {
- return index.lookupOrd(ord);
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
index 5c36fe1..a48df03 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
@@ -16,8 +16,12 @@
*/
package org.apache.lucene.search.grouping.term;
-import org.apache.lucene.index.LeafReaderContext;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
@@ -27,10 +31,6 @@ import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.UnicodeUtil;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* An implementation of {@link AbstractGroupFacetCollector} that computes grouped facets based on the indexed terms
* from DocValues.
@@ -86,12 +86,31 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
@Override
public void collect(int doc) throws IOException {
- int facetOrd = facetFieldTermsIndex.getOrd(doc);
+ if (doc > facetFieldTermsIndex.docID()) {
+ facetFieldTermsIndex.advance(doc);
+ }
+
+ int facetOrd;
+ if (doc == facetFieldTermsIndex.docID()) {
+ facetOrd = facetFieldTermsIndex.ordValue();
+ } else {
+ facetOrd = -1;
+ }
+
if (facetOrd < startFacetOrd || facetOrd >= endFacetOrd) {
return;
}
- int groupOrd = groupFieldTermsIndex.getOrd(doc);
+ if (doc > groupFieldTermsIndex.docID()) {
+ groupFieldTermsIndex.advance(doc);
+ }
+
+ int groupOrd;
+ if (doc == groupFieldTermsIndex.docID()) {
+ groupOrd = groupFieldTermsIndex.ordValue();
+ } else {
+ groupOrd = -1;
+ }
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.getValueCount()+1) + facetOrd;
if (segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
return;
@@ -206,7 +225,17 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
@Override
public void collect(int doc) throws IOException {
- int groupOrd = groupFieldTermsIndex.getOrd(doc);
+ if (doc > groupFieldTermsIndex.docID()) {
+ groupFieldTermsIndex.advance(doc);
+ }
+
+ int groupOrd;
+ if (doc == groupFieldTermsIndex.docID()) {
+ groupOrd = groupFieldTermsIndex.ordValue();
+ } else {
+ groupOrd = -1;
+ }
+
if (facetFieldNumTerms == 0) {
int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1);
if (facetPrefix != null || segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
@@ -227,12 +256,16 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
return;
}
- facetFieldDocTermOrds.setDocument(doc);
- long ord;
+ if (doc > facetFieldDocTermOrds.docID()) {
+ facetFieldDocTermOrds.advance(doc);
+ }
boolean empty = true;
- while ((ord = facetFieldDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- process(groupOrd, (int) ord);
- empty = false;
+ if (doc == facetFieldDocTermOrds.docID()) {
+ long ord;
+ while ((ord = facetFieldDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ process(groupOrd, (int) ord);
+ empty = false;
+ }
}
if (empty) {
@@ -379,4 +412,4 @@ class GroupedFacetHit {
this.groupValue = groupValue;
this.facetValue = facetValue;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
index 9292856..c7b16e9 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
@@ -19,8 +19,8 @@ package org.apache.lucene.search.grouping.term;
import java.io.IOException;
import java.util.Collection;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
@@ -30,7 +30,7 @@ import org.apache.lucene.util.SentinelIntSet;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector} that groups based on
- * field values and more specifically uses {@link org.apache.lucene.index.SortedDocValues}
+ * field values and more specifically uses {@link SortedDocValues}
* to collect grouped docs.
*
* @lucene.experimental
@@ -70,7 +70,18 @@ public class TermSecondPassGroupingCollector extends AbstractSecondPassGroupingC
@Override
protected SearchGroupDocs<BytesRef> retrieveGroup(int doc) throws IOException {
- int slot = ordSet.find(index.getOrd(doc));
+ if (doc > index.docID()) {
+ index.advance(doc);
+ }
+
+ int ord;
+ if (doc == index.docID()) {
+ ord = index.ordValue();
+ } else {
+ ord = -1;
+ }
+
+ int slot = ordSet.find(ord);
if (slot >= 0) {
return groupDocs[slot];
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
index 9bdb8f5..316676b 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
@@ -283,10 +283,13 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
final DirectoryReader r = w.getReader();
w.close();
- final NumericDocValues docIdToFieldId = MultiDocValues.getNumericValues(r, "id");
+ NumericDocValues values = MultiDocValues.getNumericValues(r, "id");
+ final int[] docIDToFieldId = new int[numDocs];
final int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < numDocs; i++) {
- int fieldId = (int) docIdToFieldId.get(i);
+ assertEquals(i, values.nextDoc());
+ int fieldId = (int) values.longValue();
+ docIDToFieldId[i] = fieldId;
fieldIdToDocID[fieldId] = i;
}
@@ -296,7 +299,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
for (int contentID = 0; contentID < 3; contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
for (ScoreDoc hit : hits) {
- int idValue = (int) docIdToFieldId.get(hit.doc);
+ int idValue = docIDToFieldId[hit.doc];
final GroupDoc gd = groupDocs[idValue];
assertEquals(gd.id, idValue);
seenIDs.add(idValue);
@@ -329,7 +332,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.length; i++) {
- actualGroupHeads[i] = (int) docIdToFieldId.get(actualGroupHeads[i]);
+ actualGroupHeads[i] = docIDToFieldId[actualGroupHeads[i]];
}
// Allows us the easily iterate and assert the actual and expected results.
Arrays.sort(expectedGroupHeads);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
index 2f51c94..4a59632 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
@@ -757,8 +757,13 @@ public class TestGrouping extends LuceneTestCase {
final DirectoryReader r = w.getReader();
w.close();
-
- final NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
+
+ NumericDocValues values = MultiDocValues.getNumericValues(r, "id");
+ int[] docIDToID = new int[r.maxDoc()];
+ for(int i=0;i<r.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToID[i] = (int) values.longValue();
+ }
DirectoryReader rBlocks = null;
Directory dirBlocks = null;
@@ -773,7 +778,7 @@ public class TestGrouping extends LuceneTestCase {
for(int contentID=0;contentID<3;contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
- int idValue = (int) docIDToID.get(hit.doc);
+ int idValue = docIDToID[hit.doc];
final GroupDoc gd = groupDocs[idValue];
seenIDs.add(idValue);
@@ -796,8 +801,6 @@ public class TestGrouping extends LuceneTestCase {
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Query lastDocInBlock = new TermQuery(new Term("groupend", "x"));
- final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
- assertNotNull(docIDToIDBlocks);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);
@@ -806,6 +809,14 @@ public class TestGrouping extends LuceneTestCase {
// means a monotonic shift in scores, so we can
// reliably remap them w/ Map:
final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
+
+ values = MultiDocValues.getNumericValues(rBlocks, "id");
+ assertNotNull(values);
+ int[] docIDToIDBlocks = new int[rBlocks.maxDoc()];
+ for(int i=0;i<rBlocks.maxDoc();i++) {
+ assertEquals(i, values.nextDoc());
+ docIDToIDBlocks[i] = (int) values.longValue();
+ }
// Tricky: must separately set .score2, because the doc
// block index was created with possible deletions!
@@ -818,11 +829,11 @@ public class TestGrouping extends LuceneTestCase {
//" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
- final GroupDoc gd = groupDocsByID[(int) docIDToIDBlocks.get(hit.doc)];
+ final GroupDoc gd = groupDocsByID[docIDToIDBlocks[hit.doc]];
assertTrue(gd.score2 == 0.0);
gd.score2 = hit.score;
- assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
- //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
+ assertEquals(gd.id, docIDToIDBlocks[hit.doc]);
+ //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks[hit.doc]);
termScoreMap.put(gd.score, gd.score2);
}
}
@@ -1005,13 +1016,13 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : groupsResult.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToID[sd.doc] + " score=" + sd.score);
}
}
if (searchIter == 14) {
for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
- System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
+ System.out.println("ID=" + docIDToID[docIDX] + " explain=" + s.explain(query, docIDX));
}
}
}
@@ -1023,7 +1034,7 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToID[sd.doc] + " score=" + sd.score);
}
}
}
@@ -1072,7 +1083,7 @@ public class TestGrouping extends LuceneTestCase {
for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
+ System.out.println(" id=" + docIDToIDBlocks[sd.doc] + " score=" + sd.score);
if (first) {
System.out.println("explain: " + sBlocks.explain(query, sd.doc));
first = false;
@@ -1237,7 +1248,7 @@ public class TestGrouping extends LuceneTestCase {
}
}
- private void assertEquals(NumericDocValues docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
+ private void assertEquals(int[] docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
if (expected == null) {
assertNull(actual);
return;
@@ -1284,8 +1295,8 @@ public class TestGrouping extends LuceneTestCase {
for(int docIDX=0;docIDX<expectedFDs.length;docIDX++) {
final FieldDoc expectedFD = (FieldDoc) expectedFDs[docIDX];
final FieldDoc actualFD = (FieldDoc) actualFDs[docIDX];
- //System.out.println(" actual doc=" + docIDtoID.get(actualFD.doc) + " score=" + actualFD.score);
- assertEquals(expectedFD.doc, docIDtoID.get(actualFD.doc));
+ //System.out.println(" actual doc=" + docIDtoID[actualFD.doc] + " score=" + actualFD.score);
+ assertEquals(expectedFD.doc, docIDtoID[actualFD.doc]);
if (testScores) {
assertEquals(expectedFD.score, actualFD.score, 0.1);
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 55f360a..07506f2 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -129,11 +129,6 @@ public class TermVectorLeafReader extends LeafReader {
}
@Override
- public Bits getDocsWithField(String field) throws IOException {
- return null;
- }
-
- @Override
public NumericDocValues getNormValues(String field) throws IOException {
return null;//Is this needed? See MemoryIndex for a way to do it.
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
index 7f79809..1b277f1 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
@@ -63,9 +63,9 @@ import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
@@ -473,11 +473,6 @@ public class WeightedSpanTermExtractor {
public NumericDocValues getNormValues(String field) throws IOException {
return super.getNormValues(FIELD_NAME);
}
-
- @Override
- public Bits getDocsWithField(String field) throws IOException {
- return super.getDocsWithField(FIELD_NAME);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java b/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
index 507d6a6..e034369 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
@@ -16,14 +16,14 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
-import java.io.IOException;
-
abstract class BaseGlobalOrdinalScorer extends Scorer {
final SortedDocValues values;
@@ -64,4 +64,4 @@ abstract class BaseGlobalOrdinalScorer extends Scorer {
protected abstract TwoPhaseIterator createTwoPhaseIterator(DocIdSetIterator approximation);
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java b/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
index 509c1e9..212d75b 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/BlockJoinSelector.java
@@ -16,7 +16,8 @@
*/
package org.apache.lucene.search.join;
-import org.apache.lucene.index.DocValues;
+import java.io.IOException;
+
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
@@ -96,37 +97,77 @@ public class BlockJoinSelector {
* one value per parent among its {@code children} using the configured
* {@code selection} type. */
public static SortedDocValues wrap(final SortedDocValues values, Type selection, BitSet parents, BitSet children) {
+ if (values.docID() != -1) {
+ throw new IllegalArgumentException("values iterator was already consumed: values.docID=" + values.docID());
+ }
return new SortedDocValues() {
+ private int ord;
+ private int docID = -1;
+
@Override
- public int getOrd(int docID) {
- assert parents.get(docID) : "this selector may only be used on parent documents";
+ public int docID() {
+ return docID;
+ }
- if (docID == 0) {
- // no children
- return -1;
+ @Override
+ public int nextDoc() throws IOException {
+ assert docID != NO_MORE_DOCS;
+
+ if (values.docID() == -1) {
+ if (values.nextDoc() == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
}
- final int firstChild = parents.prevSetBit(docID - 1) + 1;
+ if (values.docID() == NO_MORE_DOCS) {
+ docID = NO_MORE_DOCS;
+ return docID;
+ }
+
+ int nextParentDocID = parents.nextSetBit(values.docID());
+ ord = values.ordValue();
- int ord = -1;
- for (int child = children.nextSetBit(firstChild); child < docID; child = children.nextSetBit(child + 1)) {
- final int childOrd = values.getOrd(child);
- switch (selection) {
- case MIN:
- if (ord == -1) {
- ord = childOrd;
- } else if (childOrd != -1) {
- ord = Math.min(ord, childOrd);
- }
- break;
- case MAX:
- ord = Math.max(ord, childOrd);
- break;
- default:
- throw new AssertionError();
+ while (true) {
+ int childDocID = values.nextDoc();
+ assert childDocID != nextParentDocID;
+ if (childDocID > nextParentDocID) {
+ break;
+ }
+ if (children.get(childDocID) == false) {
+ continue;
}
+ if (selection == Type.MIN) {
+ ord = Math.min(ord, values.ordValue());
+ } else if (selection == Type.MAX) {
+ ord = Math.max(ord, values.ordValue());
+ } else {
+ throw new AssertionError();
+ }
+ }
+
+ docID = nextParentDocID;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= parents.length()) {
+ docID = NO_MORE_DOCS;
+ return docID;
}
+ if (target == 0) {
+ assert docID() == -1;
+ return nextDoc();
+ }
+ int prevParentDocID = parents.prevSetBit(target-1);
+ values.advance(prevParentDocID+1);
+ return nextDoc();
+ }
+
+ @Override
+ public int ordValue() {
return ord;
}
@@ -140,6 +181,10 @@ public class BlockJoinSelector {
return values.getValueCount();
}
+ @Override
+ public long cost() {
+ return values.cost();
+ }
};
}
@@ -158,53 +203,102 @@ public class BlockJoinSelector {
default:
throw new AssertionError();
}
- return wrap(values, DocValues.docsWithValue(sortedNumerics, parents.length()), selection, parents, children);
+ return wrap(values, selection, parents, children);
}
- /** Wraps the provided {@link NumericDocValues} in order to only select
- * one value per parent among its {@code children} using the configured
- * {@code selection} type. */
- public static NumericDocValues wrap(final NumericDocValues values, Bits docsWithValue, Type selection, BitSet parents, BitSet children) {
+ /** Wraps the provided {@link NumericDocValues}, iterating over only
+ * child documents, in order to only select one value per parent among
+ * its {@code children} using the configured {@code selection} type. */
+ public static NumericDocValues wrap(final NumericDocValues values, Type selection, BitSet parents, BitSet children) {
return new NumericDocValues() {
+ private int parentDocID = -1;
+ private long value;
+
@Override
- public long get(int docID) {
- assert parents.get(docID) : "this selector may only be used on parent documents";
+ public int nextDoc() throws IOException {
- if (docID == 0) {
- // no children
- return 0;
+ if (parentDocID == -1) {
+ values.nextDoc();
}
- final int firstChild = parents.prevSetBit(docID - 1) + 1;
+ while (true) {
- long value = 0;
- boolean hasValue = false;
+ // TODO: make this crazy loop more efficient
- for (int child = children.nextSetBit(firstChild); child < docID; child = children.nextSetBit(child + 1)) {
- final long childValue = values.get(child);
- final boolean childHasValue = value != 0 || docsWithValue.get(child);
+ int childDocID = values.docID();
+ if (childDocID == NO_MORE_DOCS) {
+ parentDocID = NO_MORE_DOCS;
+ return parentDocID;
+ }
+ if (children.get(childDocID) == false) {
+ values.nextDoc();
+ continue;
+ }
+
+ assert parents.get(childDocID) == false;
+
+ parentDocID = parents.nextSetBit(childDocID);
+ value = values.longValue();
+
+ while (true) {
+ childDocID = values.nextDoc();
+ assert childDocID != parentDocID;
+ if (childDocID > parentDocID) {
+ break;
+ }
- if (hasValue == false) {
- value = childValue;
- hasValue = childHasValue;
- } else if (childHasValue) {
switch (selection) {
- case MIN:
- value = Math.min(value, childValue);
- break;
- case MAX:
- value = Math.max(value, childValue);
- break;
- default:
- throw new AssertionError();
+ case MIN:
+ value = Math.min(value, values.longValue());
+ break;
+ case MAX:
+ value = Math.max(value, values.longValue());
+ break;
+ default:
+ throw new AssertionError();
}
}
+
+ break;
+ }
+
+ return parentDocID;
+ }
+
+ @Override
+ public int advance(int targetParentDocID) throws IOException {
+ if (targetParentDocID <= parentDocID) {
+ throw new IllegalArgumentException("target must be after the current document: current=" + parentDocID + " target=" + targetParentDocID);
+ }
+
+ if (targetParentDocID == 0) {
+ return nextDoc();
+ }
+
+ int firstChild = parents.prevSetBit(targetParentDocID - 1) + 1;
+ if (values.advance(firstChild) == NO_MORE_DOCS) {
+ parentDocID = NO_MORE_DOCS;
+ return parentDocID;
+ } else {
+ return nextDoc();
}
+ }
+ @Override
+ public long longValue() {
return value;
}
+
+ @Override
+ public int docID() {
+ return parentDocID;
+ }
+ @Override
+ public long cost() {
+ return values.cost();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
index ca1337d..d54543c 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DocValuesTermsCollector.java
@@ -29,7 +29,7 @@ abstract class DocValuesTermsCollector<DV> extends SimpleCollector {
@FunctionalInterface
static interface Function<R> {
- R apply(LeafReader t) throws IOException ;
+ R apply(LeafReader t) throws IOException;
}
protected DV docValues;
@@ -45,8 +45,9 @@ abstract class DocValuesTermsCollector<DV> extends SimpleCollector {
}
static Function<BinaryDocValues> binaryDocValues(String field) {
- return (ctx) -> DocValues.getBinary(ctx, field);
+ return (ctx) -> DocValues.getBinary(ctx, field);
}
+
static Function<SortedSetDocValues> sortedSetDocValues(String field) {
return (ctx) -> DocValues.getSortedSet(ctx, field);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
index ef8b3c1..3a630a1 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GenericTermsCollector.java
@@ -43,9 +43,9 @@ interface GenericTermsCollector extends Collector {
case None:
return wrap(new TermsCollector.MV(mvFunction));
case Avg:
- return new MV.Avg(mvFunction);
+ return new MV.Avg(mvFunction);
default:
- return new MV(mvFunction, mode);
+ return new MV(mvFunction, mode);
}
}
@@ -55,13 +55,31 @@ interface GenericTermsCollector extends Collector {
return new SortedSetDocValues() {
@Override
- public void setDocument(int docID) {
- target.setDocument(docID);
- out.println("\ndoc# "+docID);
+ public int docID() {
+ return target.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ int docID = target.nextDoc();
+ out.println("\nnextDoc doc# "+docID);
+ return docID;
+ }
+
+ @Override
+ public int advance(int dest) throws IOException {
+ int docID = target.advance(dest);
+ out.println("\nadvance(" + dest + ") -> doc# "+docID);
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return target.cost();
}
@Override
- public long nextOrd() {
+ public long nextOrd() throws IOException {
return target.nextOrd();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f7aa200d/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
index d80851a..9c9072d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -26,8 +28,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.LongValues;
-import java.io.IOException;
-
/**
* A collector that collects all ordinals from a specified field matching the query.
*
@@ -77,9 +77,12 @@ final class GlobalOrdinalsCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- final long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ long segmentOrd = docTermOrds.ordValue();
+ long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
collectedOrds.set(globalOrd);
}
}
@@ -99,9 +102,11 @@ final class GlobalOrdinalsCollector implements Collector {
@Override
public void collect(int doc) throws IOException {
- final long segmentOrd = docTermOrds.getOrd(doc);
- if (segmentOrd != -1) {
- collectedOrds.set(segmentOrd);
+ if (doc > docTermOrds.docID()) {
+ docTermOrds.advance(doc);
+ }
+ if (doc == docTermOrds.docID()) {
+ collectedOrds.set(docTermOrds.ordValue());
}
}