You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2017/05/04 03:30:55 UTC
lucene-solr:branch_6x: SOLR-10596: fix unique/hll docvalue iterator
reuse
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x d7583ba67 -> 3546fc3f5
SOLR-10596: fix unique/hll docvalue iterator reuse
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3546fc3f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3546fc3f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3546fc3f
Branch: refs/heads/branch_6x
Commit: 3546fc3f5826526ca1ca82086732d7a0ab0c1076
Parents: d7583ba
Author: yonik <yo...@apache.org>
Authored: Wed May 3 23:04:33 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Wed May 3 23:05:10 2017 -0400
----------------------------------------------------------------------
.../solr/search/facet/FacetProcessor.java | 2 +-
.../org/apache/solr/search/facet/HLLAgg.java | 6 +++---
.../org/apache/solr/search/facet/SlotAcc.java | 2 +-
.../org/apache/solr/search/facet/UniqueAgg.java | 6 +++---
.../solr/search/facet/UniqueMultiDvSlotAcc.java | 21 +++++++++++++-------
.../search/facet/UniqueMultivaluedSlotAcc.java | 5 +++--
.../search/facet/UniqueSinglevaluedSlotAcc.java | 20 ++++++++++++++-----
.../apache/solr/search/facet/UniqueSlotAcc.java | 6 +++---
.../solr/search/facet/TestJsonFacets.java | 9 +++++++++
9 files changed, 52 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
index d5a4e23..27cdaec 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
@@ -312,7 +312,7 @@ public abstract class FacetProcessor<FacetRequestT extends FacetRequest> {
}
// note: only called by enum/stream prior to collect
- void resetStats() {
+ void resetStats() throws IOException {
countAcc.reset();
for (SlotAcc acc : accs) {
acc.reset();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
index 9e4ec8a..adb9e8c 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
@@ -53,16 +53,16 @@ public class HLLAgg extends StrAggValueSource {
SchemaField sf = fcontext.qcontext.searcher().getSchema().getField(getArg());
if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
if (sf.hasDocValues()) {
- return new UniqueMultiDvSlotAcc(fcontext, getArg(), numSlots, fcontext.isShard() ? factory : null);
+ return new UniqueMultiDvSlotAcc(fcontext, sf, numSlots, fcontext.isShard() ? factory : null);
} else {
- return new UniqueMultivaluedSlotAcc(fcontext, getArg(), numSlots, fcontext.isShard() ? factory : null);
+ return new UniqueMultivaluedSlotAcc(fcontext, sf, numSlots, fcontext.isShard() ? factory : null);
}
} else {
if (sf.getType().getNumberType() != null) {
// always use hll here since we don't know how many values there are?
return new NumericAcc(fcontext, getArg(), numSlots);
} else {
- return new UniqueSinglevaluedSlotAcc(fcontext, getArg(), numSlots, fcontext.isShard() ? factory : null);
+ return new UniqueSinglevaluedSlotAcc(fcontext, sf, numSlots, fcontext.isShard() ? factory : null);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
index 4bb7ca9..168ca8d 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
@@ -94,7 +94,7 @@ public abstract class SlotAcc implements Closeable {
}
}
- public abstract void reset();
+ public abstract void reset() throws IOException;
public abstract void resize(Resizer resizer);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
index 8e2c603..6c18ae4 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
@@ -44,15 +44,15 @@ public class UniqueAgg extends StrAggValueSource {
SchemaField sf = fcontext.qcontext.searcher().getSchema().getField(getArg());
if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
if (sf.hasDocValues()) {
- return new UniqueMultiDvSlotAcc(fcontext, getArg(), numSlots, null);
+ return new UniqueMultiDvSlotAcc(fcontext, sf, numSlots, null);
} else {
- return new UniqueMultivaluedSlotAcc(fcontext, getArg(), numSlots, null);
+ return new UniqueMultivaluedSlotAcc(fcontext, sf, numSlots, null);
}
} else {
if (sf.getType().getNumberType() != null) {
return new NumericAcc(fcontext, getArg(), numSlots);
} else {
- return new UniqueSinglevaluedSlotAcc(fcontext, getArg(), numSlots, null);
+ return new UniqueSinglevaluedSlotAcc(fcontext, sf, numSlots, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
index 4c29753..1249204 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java
@@ -25,19 +25,23 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
-import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.schema.SchemaField;
class UniqueMultiDvSlotAcc extends UniqueSlotAcc {
- final SortedSetDocValues topLevel;
- final SortedSetDocValues[] subDvs;
- final MultiDocValues.OrdinalMap ordMap;
+ SortedSetDocValues topLevel;
+ SortedSetDocValues[] subDvs;
+ MultiDocValues.OrdinalMap ordMap;
LongValues toGlobal;
SortedSetDocValues subDv;
- public UniqueMultiDvSlotAcc(FacetContext fcontext, String field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
+ public UniqueMultiDvSlotAcc(FacetContext fcontext, SchemaField field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
super(fcontext, field, numSlots, factory);
- SolrIndexSearcher searcher = fcontext.qcontext.searcher();
- topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, searcher.getSchema().getField(field), null);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, field, null);
nTerms = (int) topLevel.getValueCount();
if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) {
ordMap = ((MultiDocValues.MultiSortedSetDocValues) topLevel).mapping;
@@ -55,6 +59,9 @@ class UniqueMultiDvSlotAcc extends UniqueSlotAcc {
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
+ if (topLevel == null) {
+ reset();
+ }
super.setNextReader(readerContext);
if (subDvs != null) {
subDv = subDvs[readerContext.ord];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/UniqueMultivaluedSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultivaluedSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultivaluedSlotAcc.java
index 56a498e..10adcf0 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultivaluedSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultivaluedSlotAcc.java
@@ -21,16 +21,17 @@ import java.io.IOException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
class UniqueMultivaluedSlotAcc extends UniqueSlotAcc implements UnInvertedField.Callback {
private UnInvertedField uif;
private UnInvertedField.DocToTerm docToTerm;
- public UniqueMultivaluedSlotAcc(FacetContext fcontext, String field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
+ public UniqueMultivaluedSlotAcc(FacetContext fcontext, SchemaField field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
super(fcontext, field, numSlots, factory);
SolrIndexSearcher searcher = fcontext.qcontext.searcher();
- uif = UnInvertedField.getUnInvertedField(field, searcher);
+ uif = UnInvertedField.getUnInvertedField(field.getName(), searcher);
docToTerm = uif.new DocToTerm();
fcontext.qcontext.addCloseHook(this); // TODO: find way to close accumulators instead of using close hook?
nTerms = uif.numTerms();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
index c67fd47..954e55b 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
@@ -25,19 +25,26 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
class UniqueSinglevaluedSlotAcc extends UniqueSlotAcc {
- final SortedDocValues topLevel;
- final SortedDocValues[] subDvs;
- final MultiDocValues.OrdinalMap ordMap;
+ SortedDocValues topLevel;
+ SortedDocValues[] subDvs;
+ MultiDocValues.OrdinalMap ordMap;
LongValues toGlobal;
SortedDocValues subDv;
- public UniqueSinglevaluedSlotAcc(FacetContext fcontext, String field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
+ public UniqueSinglevaluedSlotAcc(FacetContext fcontext, SchemaField field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
super(fcontext, field, numSlots, factory);
+ // let setNextReader lazily call reset(), that way an extra call to reset() after creation won't matter
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
SolrIndexSearcher searcher = fcontext.qcontext.searcher();
- topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, searcher.getSchema().getField(field), null);
+ topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null);
nTerms = topLevel.getValueCount();
if (topLevel instanceof MultiDocValues.MultiSortedDocValues) {
ordMap = ((MultiDocValues.MultiSortedDocValues)topLevel).mapping;
@@ -55,6 +62,9 @@ class UniqueSinglevaluedSlotAcc extends UniqueSlotAcc {
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
+ if (topLevel == null) {
+ reset();
+ }
super.setNextReader(readerContext);
if (subDvs != null) {
subDv = subDvs[readerContext.ord];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
index ae542ac..607b067 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
@@ -37,15 +37,15 @@ abstract class UniqueSlotAcc extends SlotAcc {
int[] counts; // populated with the cardinality once
int nTerms;
- public UniqueSlotAcc(FacetContext fcontext, String field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
+ public UniqueSlotAcc(FacetContext fcontext, SchemaField field, int numSlots, HLLAgg.HLLFactory factory) throws IOException {
super(fcontext);
this.factory = factory;
arr = new FixedBitSet[numSlots];
- this.field = fcontext.searcher.getSchema().getField(field);
+ this.field = field;
}
@Override
- public void reset() {
+ public void reset() throws IOException {
counts = null;
for (FixedBitSet bits : arr) {
if (bits == null) continue;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3546fc3f/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 3f31cca..4fcee67 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -1360,6 +1360,15 @@ public class TestJsonFacets extends SolrTestCaseHS {
"}"
);
+ // test acc reuse (i.e. reset() method). This is normally used for stats that are not calculated in the first phase,
+ // currently non-sorting stats.
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{type:terms, field:'${cat_s}', facet:{h:'hll(${where_s})'} }}"
+ )
+ , "facets=={ 'count':6, " +
+ "'f1':{ buckets:[{val:B, count:3, h:2},{val:A, count:2, h:2}] } } "
+ );
+
}
@Test