You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/09/07 09:57:23 UTC
[35/50] [abbrv] lucene-solr:apiv2: SOLR-9142: json.facet: new
method=dvhash which works on terms. Also: (1) method=stream now requires you
set sort=index asc to work (2) faceting on numerics with prefix or mincount=0
will give you an error (3) refactored
SOLR-9142: json.facet: new method=dvhash which works on terms. Also:
(1) method=stream now requires you set sort=index asc to work
(2) faceting on numerics with prefix or mincount=0 will give you an error
(3) refactored similar findTopSlots into one common one in FacetFieldProcessor
(4) new DocSet.collectSortedDocSet utility
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7b5df8a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7b5df8a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7b5df8a1
Branch: refs/heads/apiv2
Commit: 7b5df8a10391f5b824e8ea1793917ff60b64b8a8
Parents: 23825b2
Author: David Smiley <ds...@apache.org>
Authored: Wed Aug 31 16:54:24 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Wed Aug 31 16:54:24 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 11 +
.../java/org/apache/solr/search/DocSetUtil.java | 33 ++
.../apache/solr/search/facet/FacetField.java | 68 ++--
.../solr/search/facet/FacetFieldProcessor.java | 150 ++++++++-
.../facet/FacetFieldProcessorByArray.java | 144 +--------
.../facet/FacetFieldProcessorByHashNumeric.java | 324 ++++++++++---------
.../org/apache/solr/search/facet/SlotAcc.java | 15 +-
.../solr/search/facet/TestJsonFacets.java | 47 ++-
8 files changed, 449 insertions(+), 343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cc90e6e..cc28449 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -66,6 +66,15 @@ Jetty 9.3.8.v20160314
Detailed Change List
----------------------
+Upgrade Notes
+----------------------
+
+* If you use the JSON Facet API (json.facet) with method=stream, you must now set sort='index asc' to get the streaming
+behavior; otherwise it won't stream. Reminder: "method" is a hint that doesn't change defaults of other parameters.
+
+* If you use the JSON Facet API (json.facet) to facet on a numeric field and if you use mincount=0 or if you set the
+prefix, then you will now get an error as these options are incompatible with numeric faceting.
+
New Features
----------------------
@@ -105,6 +114,8 @@ Optimizations
* SOLR-9452: JsonRecordReader should not deep copy document before handler.handle(). (noble, shalin)
+* SOLR-9142: JSON Facet API: new method=dvhash can be chosen for fields with high cardinality. (David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/DocSetUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/DocSetUtil.java b/solr/core/src/java/org/apache/solr/search/DocSetUtil.java
index cc2393d..b7545e6 100644
--- a/solr/core/src/java/org/apache/solr/search/DocSetUtil.java
+++ b/solr/core/src/java/org/apache/solr/search/DocSetUtil.java
@@ -17,10 +17,12 @@
package org.apache.solr.search;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -29,7 +31,9 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.Bits;
@@ -208,4 +212,33 @@ public class DocSetUtil {
return new SortedIntDocSet(docs);
}
+ public static void collectSortedDocSet(DocSet docs, IndexReader reader, Collector collector) throws IOException {
+ // TODO add SortedDocSet sub-interface and take that.
+ // TODO collectUnsortedDocSet: iterate segment, then all docSet per segment.
+
+ final List<LeafReaderContext> leaves = reader.leaves();
+ final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
+ int segBase = 0;
+ int segMax;
+ int adjustedMax = 0;
+ LeafReaderContext ctx = null;
+ LeafCollector leafCollector = null;
+ for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+ final int doc = docsIt.nextDoc();
+ if (doc >= adjustedMax) {
+ do {
+ ctx = ctxIt.next();
+ segBase = ctx.docBase;
+ segMax = ctx.reader().maxDoc();
+ adjustedMax = segBase + segMax;
+ } while (doc >= adjustedMax);
+ leafCollector = collector.getLeafCollector(ctx);
+ }
+ if (doc < segBase) {
+ throw new IllegalStateException("algorithm expects sorted DocSet but wasn't: " + docs.getClass());
+ }
+ leafCollector.collect(doc - segBase); // per-seg collectors
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
index 9cc5420..4d56513 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
@@ -66,28 +66,29 @@ public class FacetField extends FacetRequestSorted {
}
public enum FacetMethod {
- DV, // DocValues
- UIF, // UnInvertedField
- ENUM,
- STREAM,
+ DV, // DocValues, collect into ordinal array
+ UIF, // UnInvertedField, collect into ordinal array
+ DVHASH, // DocValues, collect into hash
+ ENUM, // TermsEnum then intersect DocSet (stream-able)
+ STREAM, // presently equivalent to ENUM
SMART,
;
public static FacetMethod fromString(String method) {
- if (method == null || method.length()==0) return null;
- if ("dv".equals(method)) {
- return DV;
- } else if ("uif".equals(method)) {
- return UIF;
- } else if ("enum".equals(method)) {
- return ENUM;
- } else if ("smart".equals(method)) {
- return SMART;
- } else if ("stream".equals(method)) {
- return STREAM;
+ if (method == null || method.length()==0) return DEFAULT_METHOD;
+ switch (method) {
+ case "dv": return DV;
+ case "uif": return UIF;
+ case "dvhash": return DVHASH;
+ case "enum": return ENUM;
+ case "stream": return STREAM; // TODO replace with enum?
+ case "smart": return SMART;
+ default:
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown FacetField method " + method);
}
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown FacetField method " + method);
}
+
+ static FacetMethod DEFAULT_METHOD = SMART; // non-final for tests to vary
}
@Override
@@ -96,21 +97,42 @@ public class FacetField extends FacetRequestSorted {
FieldType ft = sf.getType();
boolean multiToken = sf.multiValued() || ft.multiValuedFieldCache();
- if (method == FacetMethod.ENUM && sf.indexed()) {
- throw new UnsupportedOperationException();
- } else if (method == FacetMethod.STREAM && sf.indexed()) {
+ LegacyNumericType ntype = ft.getNumericType();
+ // ensure we can support the requested options for numeric faceting:
+ if (ntype != null) {
+ if (prefix != null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Doesn't make sense to set facet prefix on a numeric field");
+ }
+ if (mincount == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Numeric fields do not support facet mincount=0; try indexing as terms");
+ // TODO if indexed=true then we could add support
+ }
+ }
+
+ // TODO auto-pick ENUM/STREAM SOLR-9351 when index asc and DocSet cardinality is *not* much smaller than term cardinality
+ if (method == FacetMethod.ENUM) {// at the moment these two are the same
+ method = FacetMethod.STREAM;
+ }
+ if (method == FacetMethod.STREAM && sf.indexed() &&
+ "index".equals(sortVariable) && sortDirection == SortDirection.asc) {
return new FacetFieldProcessorByEnumTermsStream(fcontext, this, sf);
}
- LegacyNumericType ntype = ft.getNumericType();
+ // TODO if method=UIF and not single-valued numerics then simply choose that now? TODO add FieldType.getDocValuesType()
if (!multiToken) {
- if (ntype != null) {
- // single valued numeric (docvalues or fieldcache)
+ if (mincount > 0 && prefix == null && (ntype != null || method == FacetMethod.DVHASH)) {
+ // TODO can we auto-pick for strings when term cardinality is much greater than DocSet cardinality?
+ // or if we don't know cardinality but DocSet size is very small
return new FacetFieldProcessorByHashNumeric(fcontext, this, sf);
- } else {
+ } else if (ntype == null) {
// single valued string...
return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Couldn't pick facet algorithm for field " + sf);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
index a737321..3c1a40c 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
@@ -18,12 +18,18 @@
package org.apache.solr.search.facet;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.function.BiPredicate;
+import java.util.function.Function;
+import java.util.function.IntFunction;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocSet;
@@ -57,7 +63,7 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount);
}
- // This is used to create accs for second phase (or to create accs for all aggs)
+ /** This is used to create accs for second phase (or to create accs for all aggs) */
@Override
protected void createAccs(int docCount, int slotCount) throws IOException {
if (accMap == null) {
@@ -195,7 +201,140 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
}
- void fillBucket(SimpleOrderedMap<Object> target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException {
+ /** Processes the collected data to finds the top slots, and composes it in the response NamedList. */
+ SimpleOrderedMap<Object> findTopSlots(final int numSlots, final int slotCardinality,
+ IntFunction<Comparable> bucketValFromSlotNumFunc,
+ Function<Comparable, String> fieldQueryValFunc) throws IOException {
+ int numBuckets = 0;
+ List<Object> bucketVals = null;
+ if (freq.numBuckets && fcontext.isShard()) {
+ bucketVals = new ArrayList<>(100);
+ }
+
+ final int off = fcontext.isShard() ? 0 : (int) freq.offset;
+ // add a modest amount of over-request if this is a shard request
+ final int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
+
+ final int sortMul = freq.sortDirection.getMultiplier();
+
+ int maxTopVals = (int) (lim >= 0 ? (long) off + lim : Integer.MAX_VALUE - 1);
+ maxTopVals = Math.min(maxTopVals, slotCardinality);
+ final SlotAcc sortAcc = this.sortAcc, indexOrderAcc = this.indexOrderAcc;
+ final BiPredicate<Slot,Slot> orderPredicate;
+ if (indexOrderAcc != null && indexOrderAcc != sortAcc) {
+ orderPredicate = (a, b) -> {
+ int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
+ return cmp == 0 ? (indexOrderAcc.compare(a.slot, b.slot) > 0) : cmp < 0;
+ };
+ } else {
+ orderPredicate = (a, b) -> {
+ int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
+ return cmp == 0 ? b.slot < a.slot : cmp < 0;
+ };
+ }
+ final PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxTopVals) {
+ @Override
+ protected boolean lessThan(Slot a, Slot b) { return orderPredicate.test(a, b); }
+ };
+
+ // note: We avoid object allocation by having a Slot and re-using the 'bottom'.
+ Slot bottom = null;
+ Slot scratchSlot = new Slot();
+ for (int slotNum = 0; slotNum < numSlots; slotNum++) {
+ // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets)
+ if (effectiveMincount > 0 && countAcc.getCount(slotNum) < effectiveMincount) {
+ continue;
+ }
+
+ numBuckets++;
+ if (bucketVals != null && bucketVals.size()<100) {
+ Object val = bucketValFromSlotNumFunc.apply(slotNum);
+ bucketVals.add(val);
+ }
+
+ if (bottom != null) {
+ scratchSlot.slot = slotNum; // scratchSlot is only used to hold this slotNum for the following line
+ if (orderPredicate.test(bottom, scratchSlot)) {
+ bottom.slot = slotNum;
+ bottom = queue.updateTop();
+ }
+ } else if (lim > 0) {
+ // queue not full
+ Slot s = new Slot();
+ s.slot = slotNum;
+ queue.add(s);
+ if (queue.size() >= maxTopVals) {
+ bottom = queue.top();
+ }
+ }
+ }
+
+ assert queue.size() <= numBuckets;
+
+ SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
+ if (freq.numBuckets) {
+ if (!fcontext.isShard()) {
+ res.add("numBuckets", numBuckets);
+ } else {
+ SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
+ map.add("numBuckets", numBuckets);
+ map.add("vals", bucketVals);
+ res.add("numBuckets", map);
+ }
+ }
+
+ FacetDebugInfo fdebug = fcontext.getDebugInfo();
+ if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
+
+ if (freq.allBuckets) {
+ SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
+ // countAcc.setValues(allBuckets, allBucketsSlot);
+ allBuckets.add("count", allBucketsAcc.getSpecialCount());
+ allBucketsAcc.setValues(allBuckets, -1); // -1 slotNum is unused for SpecialSlotAcc
+ // allBuckets currently doesn't execute sub-facets (because it doesn't change the domain?)
+ res.add("allBuckets", allBuckets);
+ }
+
+ if (freq.missing) {
+ // TODO: it would be more efficient to build up a missing DocSet if we need it here anyway.
+ SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
+ fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
+ res.add("missing", missingBucket);
+ }
+
+ // if we are deep paging, we don't have to order the highest "offset" counts.
+ int collectCount = Math.max(0, queue.size() - off);
+ assert collectCount <= lim;
+ int[] sortedSlots = new int[collectCount];
+ for (int i = collectCount - 1; i >= 0; i--) {
+ sortedSlots[i] = queue.pop().slot;
+ }
+
+ ArrayList<SimpleOrderedMap> bucketList = new ArrayList<>(collectCount);
+ res.add("buckets", bucketList);
+
+ boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
+
+ for (int slotNum : sortedSlots) {
+ SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
+ Comparable val = bucketValFromSlotNumFunc.apply(slotNum);
+ bucket.add("val", val);
+
+ Query filter = needFilter ? sf.getType().getFieldQuery(null, sf, fieldQueryValFunc.apply(val)) : null;
+
+ fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter);
+
+ bucketList.add(bucket);
+ }
+
+ return res;
+ }
+
+ private static class Slot {
+ int slot;
+ }
+
+ private void fillBucket(SimpleOrderedMap<Object> target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException {
target.add("count", count);
if (count <= 0 && !freq.processEmpty) return;
@@ -272,13 +411,6 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
}
- static class Slot {
- int slot;
- public int tiebreakCompare(int slotA, int slotB) {
- return slotB - slotA;
- }
- }
-
static class SpecialSlotAcc extends SlotAcc {
SlotAcc collectAcc;
SlotAcc[] otherAccs;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java
index 10aa4d9..767bb55 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java
@@ -18,19 +18,15 @@
package org.apache.solr.search.facet;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
/**
- * Base class for DV/UIF accumulating counts into an array by ordinal.
+ * Base class for DV/UIF accumulating counts into an array by ordinal. It's
+ * for {@link org.apache.lucene.index.SortedDocValues} and {@link org.apache.lucene.index.SortedSetDocValues} only.
* It can handle terms (strings), not numbers directly but those encoded as terms, and is multi-valued capable.
*/
abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
@@ -57,11 +53,10 @@ abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
@Override
public void process() throws IOException {
super.process();
- sf = fcontext.searcher.getSchema().getField(freq.field);
- response = getFieldCacheCounts();
+ response = calcFacets();
}
- private SimpleOrderedMap<Object> getFieldCacheCounts() throws IOException {
+ private SimpleOrderedMap<Object> calcFacets() throws IOException {
String prefix = freq.prefix;
if (prefix == null || prefix.length() == 0) {
prefixRef = null;
@@ -86,128 +81,15 @@ abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
collectDocs();
- return findTopSlots();
- }
-
- private SimpleOrderedMap<Object> findTopSlots() throws IOException {
- SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
-
- int numBuckets = 0;
- List<Object> bucketVals = null;
- if (freq.numBuckets && fcontext.isShard()) {
- bucketVals = new ArrayList<>(100);
- }
-
- int off = fcontext.isShard() ? 0 : (int) freq.offset;
- // add a modest amount of over-request if this is a shard request
- int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
-
- int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1);
- maxsize = Math.min(maxsize, nTerms);
-
- final int sortMul = freq.sortDirection.getMultiplier();
- final SlotAcc sortAcc = this.sortAcc;
-
- PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxsize) {
- @Override
- protected boolean lessThan(Slot a, Slot b) {
- int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
- return cmp == 0 ? b.slot < a.slot : cmp < 0;
- }
- };
-
- Slot bottom = null;
- for (int i = 0; i < nTerms; i++) {
- // screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets)
- if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) {
- continue;
- }
-
- numBuckets++;
- if (bucketVals != null && bucketVals.size()<100) {
- int ord = startTermIndex + i;
- BytesRef br = lookupOrd(ord);
- Object val = sf.getType().toObject(sf, br);
- bucketVals.add(val);
- }
-
- if (bottom != null) {
- if (sortAcc.compare(bottom.slot, i) * sortMul < 0) {
- bottom.slot = i;
- bottom = queue.updateTop();
- }
- } else if (lim > 0) {
- // queue not full
- Slot s = new Slot();
- s.slot = i;
- queue.add(s);
- if (queue.size() >= maxsize) {
- bottom = queue.top();
- }
- }
- }
-
- if (freq.numBuckets) {
- if (!fcontext.isShard()) {
- res.add("numBuckets", numBuckets);
- } else {
- SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
- map.add("numBuckets", numBuckets);
- map.add("vals", bucketVals);
- res.add("numBuckets", map);
- }
- }
-
- FacetDebugInfo fdebug = fcontext.getDebugInfo();
- if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
-
- // if we are deep paging, we don't have to order the highest "offset" counts.
- int collectCount = Math.max(0, queue.size() - off);
- assert collectCount <= lim;
- int[] sortedSlots = new int[collectCount];
- for (int i = collectCount - 1; i >= 0; i--) {
- sortedSlots[i] = queue.pop().slot;
- }
-
- if (freq.allBuckets) {
- SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
- allBuckets.add("count", allBucketsAcc.getSpecialCount());
- if (allBucketsAcc != null) {
- allBucketsAcc.setValues(allBuckets, allBucketsSlot);
- }
- res.add("allBuckets", allBuckets);
- }
-
- ArrayList<SimpleOrderedMap<Object>> bucketList = new ArrayList<>(collectCount);
- res.add("buckets", bucketList);
-
- // TODO: do this with a callback instead?
- boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
-
- for (int slotNum : sortedSlots) {
- SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
-
- // get the ord of the slot...
- int ord = startTermIndex + slotNum;
-
- BytesRef br = lookupOrd(ord);
- Object val = sf.getType().toObject(sf, br);
-
- bucket.add("val", val);
-
- TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null;
- fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter);
-
- bucketList.add(bucket);
- }
-
- if (freq.missing) {
- SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
- fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
- res.add("missing", missingBucket);
- }
-
- return res;
+ return super.findTopSlots(nTerms, nTerms,
+ slotNum -> { // getBucketValFromSlotNum
+ try {
+ return (Comparable) sf.getType().toObject(sf, lookupOrd(slotNum + startTermIndex));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ Object::toString); // getFieldQueryVal
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java
index 842df20..6d5aec5 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java
@@ -17,25 +17,37 @@
package org.apache.solr.search.facet;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import java.text.ParseException;
+import java.util.function.IntFunction;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LongValues;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocSetUtil;
/**
- * Facets numbers into a hash table.
- * It currently only works with {@link NumericDocValues} (single-valued).
+ * Facets numbers into a hash table. The number is either a raw numeric DocValues value, or
+ * a term global ordinal integer.
+ * Limitations:
+ * <ul>
+ * <li>doesn't handle multiValued, but could easily be added</li>
+ * <li>doesn't handle prefix, but could easily be added</li>
+ * <li>doesn't handle mincount==0 -- you're better off with an array alg</li>
+ * </ul>
*/
+// TODO rename: FacetFieldProcessorByHashDV
class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
static int MAXIMUM_STARTING_TABLE_SIZE=1024; // must be a power of two, non-final to support setting by tests
@@ -44,7 +56,6 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
static final float LOAD_FACTOR = 0.7f;
- long numAdds;
long[] vals;
int[] counts; // maintain the counts here since we need them to tell if there was actually a value anyway
int[] oldToNewMapping;
@@ -82,7 +93,6 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
rehash();
}
- numAdds++;
int h = hash(val);
for (int slot = h & (vals.length-1); ;slot = (slot + ((h>>7)|1)) & (vals.length-1)) {
int count = counts[slot];
@@ -135,29 +145,93 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
}
+ /** A hack instance of Calc for Term ordinals in DocValues. */
+ // TODO consider making FacetRangeProcessor.Calc facet top level; then less of a hack?
+ private class TermOrdCalc extends FacetRangeProcessor.Calc {
+
+ IntFunction<BytesRef> lookupOrdFunction; // set in collectDocs()!
+
+ TermOrdCalc() throws IOException {
+ super(sf);
+ }
+
+ @Override
+ public long bitsToSortableBits(long globalOrd) {
+ return globalOrd;
+ }
+
+ /** To be returned in "buckets"/"val" */
+ @Override
+ public Comparable bitsToValue(long globalOrd) {
+ BytesRef bytesRef = lookupOrdFunction.apply((int) globalOrd);
+ // note FacetFieldProcessorByArray.findTopSlots also calls SchemaFieldType.toObject
+ return sf.getType().toObject(sf, bytesRef).toString();
+ }
+
+ @Override
+ public String formatValue(Comparable val) {
+ return (String) val;
+ }
+
+ @Override
+ protected Comparable parseStr(String rawval) throws ParseException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected Comparable parseAndAddGap(Comparable value, String gap) throws ParseException {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ FacetRangeProcessor.Calc calc;
+ LongCounts table;
int allBucketsSlot = -1;
FacetFieldProcessorByHashNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) {
super(fcontext, freq, sf);
+ if (freq.mincount == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ getClass()+" doesn't support mincount=0");
+ }
+ if (freq.prefix != null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ getClass()+" doesn't support prefix"); // yet, but it could
+ }
+ FieldInfo fieldInfo = fcontext.searcher.getLeafReader().getFieldInfos().fieldInfo(sf.getName());
+ if (fieldInfo != null &&
+ fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
+ fieldInfo.getDocValuesType() != DocValuesType.SORTED) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ getClass()+" only support single valued number/string with docValues");
+ }
}
@Override
public void process() throws IOException {
super.process();
response = calcFacets();
+ table = null;//gc
}
private SimpleOrderedMap<Object> calcFacets() throws IOException {
- final FacetRangeProcessor.Calc calc = FacetRangeProcessor.getNumericCalc(sf);
+ if (sf.getType().getNumericType() != null) {
+ calc = FacetRangeProcessor.getNumericCalc(sf);
+ } else {
+ calc = new TermOrdCalc(); // kind of a hack
+ }
- // TODO: it would be really nice to know the number of unique values!!!!
+ // TODO: Use the number of indexed terms, if present, as an estimate!
+ // Even for NumericDocValues, we could check for a terms index for an estimate.
+ // Our estimation should aim high to avoid expensive rehashes.
int possibleValues = fcontext.base.size();
// size smaller tables so that no resize will be necessary
int currHashSize = BitUtil.nextHighestPowerOfTwo((int) (possibleValues * (1 / LongCounts.LOAD_FACTOR) + 1));
currHashSize = Math.min(currHashSize, MAXIMUM_STARTING_TABLE_SIZE);
- final LongCounts table = new LongCounts(currHashSize) {
+ table = new LongCounts(currHashSize) {
@Override
protected void rehash() {
super.rehash();
@@ -166,9 +240,19 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
}
};
- int numSlots = currHashSize;
+ // note: these methods/phases align with FacetFieldProcessorByArray's
- int numMissing = 0;
+ createCollectAcc();
+
+ collectDocs();
+
+ return super.findTopSlots(table.numSlots(), table.cardinality(),
+ slotNum -> calc.bitsToValue(table.vals[slotNum]), // getBucketValFromSlotNum
+ val -> calc.formatValue(val)); // getFieldQueryVal
+ }
+
+ private void createCollectAcc() throws IOException {
+ int numSlots = table.numSlots();
if (freq.allBuckets) {
allBucketsSlot = numSlots++;
@@ -238,160 +322,80 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
};
// we set the countAcc & indexAcc first so generic ones won't be created for us.
- createCollectAcc(fcontext.base.size(), numSlots);
+ super.createCollectAcc(fcontext.base.size(), numSlots);
if (freq.allBuckets) {
allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0);
}
+ }
- NumericDocValues values = null;
- Bits docsWithField = null;
-
- // TODO: factor this code out so it can be shared...
- final List<LeafReaderContext> leaves = fcontext.searcher.getIndexReader().leaves();
- final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
- LeafReaderContext ctx = null;
- int segBase = 0;
- int segMax;
- int adjustedMax = 0;
- for (DocIterator docsIt = fcontext.base.iterator(); docsIt.hasNext(); ) {
- final int doc = docsIt.nextDoc();
- if (doc >= adjustedMax) {
- do {
- ctx = ctxIt.next();
- segBase = ctx.docBase;
- segMax = ctx.reader().maxDoc();
- adjustedMax = segBase + segMax;
- } while (doc >= adjustedMax);
- assert doc >= ctx.docBase;
- setNextReaderFirstPhase(ctx);
-
- values = DocValues.getNumeric(ctx.reader(), sf.getName());
- docsWithField = DocValues.getDocsWithField(ctx.reader(), sf.getName());
- }
-
- int segDoc = doc - segBase;
- long val = values.get(segDoc);
- if (val != 0 || docsWithField.get(segDoc)) {
- int slot = table.add(val); // this can trigger a rehash rehash
-
- // countAcc.incrementCount(slot, 1);
- // our countAcc is virtual, so this is not needed
-
- collectFirstPhase(segDoc, slot);
- }
- }
-
- //
- // collection done, time to find the top slots
- //
-
- int numBuckets = 0;
- List<Object> bucketVals = null;
- if (freq.numBuckets && fcontext.isShard()) {
- bucketVals = new ArrayList<>(100);
- }
-
- int off = fcontext.isShard() ? 0 : (int) freq.offset;
- // add a modest amount of over-request if this is a shard request
- int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
-
- int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1);
- maxsize = Math.min(maxsize, table.cardinality);
-
- final int sortMul = freq.sortDirection.getMultiplier();
-
- PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxsize) {
- @Override
- protected boolean lessThan(Slot a, Slot b) {
- // TODO: sort-by-index-order
- int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
- return cmp == 0 ? (indexOrderAcc.compare(a.slot, b.slot) > 0) : cmp < 0;
- }
- };
-
- // TODO: create a countAcc that wrapps the table so we can reuse more code?
-
- Slot bottom = null;
- for (int i=0; i<table.counts.length; i++) {
- int count = table.counts[i];
- if (count < effectiveMincount) {
- // either not a valid slot, or count not high enough
- continue;
- }
- numBuckets++; // can be different from the table cardinality if mincount > 1
-
- long val = table.vals[i];
- if (bucketVals != null && bucketVals.size()<100) {
- bucketVals.add( calc.bitsToValue(val) );
- }
-
- if (bottom == null) {
- bottom = new Slot();
- }
- bottom.slot = i;
-
- bottom = queue.insertWithOverflow(bottom);
- }
-
- SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
- if (freq.numBuckets) {
- if (!fcontext.isShard()) {
- res.add("numBuckets", numBuckets);
- } else {
- SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
- map.add("numBuckets", numBuckets);
- map.add("vals", bucketVals);
- res.add("numBuckets", map);
- }
- }
-
- FacetDebugInfo fdebug = fcontext.getDebugInfo();
- if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
-
- if (freq.allBuckets) {
- SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
- // countAcc.setValues(allBuckets, allBucketsSlot);
- allBuckets.add("count", table.numAdds);
- allBucketsAcc.setValues(allBuckets, -1);
- // allBuckets currently doesn't execute sub-facets (because it doesn't change the domain?)
- res.add("allBuckets", allBuckets);
- }
-
- if (freq.missing) {
- // TODO: it would be more efficient to buid up a missing DocSet if we need it here anyway.
-
- SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
- fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
- res.add("missing", missingBucket);
- }
-
- // if we are deep paging, we don't have to order the highest "offset" counts.
- int collectCount = Math.max(0, queue.size() - off);
- assert collectCount <= lim;
- int[] sortedSlots = new int[collectCount];
- for (int i = collectCount - 1; i >= 0; i--) {
- sortedSlots[i] = queue.pop().slot;
+ private void collectDocs() throws IOException {
+ if (calc instanceof TermOrdCalc) { // Strings
+
+ // TODO support SortedSetDocValues
+ SortedDocValues globalDocValues = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null);
+ ((TermOrdCalc)calc).lookupOrdFunction = globalDocValues::lookupOrd;
+
+ DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {
+ SortedDocValues docValues = globalDocValues; // this segment/leaf. NN
+ LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN
+
+ @Override public boolean needsScores() { return false; }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
+ setNextReaderFirstPhase(ctx);
+ if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
+ MultiDocValues.MultiSortedDocValues multiDocValues = (MultiDocValues.MultiSortedDocValues) globalDocValues;
+ docValues = multiDocValues.values[ctx.ord];
+ toGlobal = multiDocValues.mapping.getGlobalOrds(ctx.ord);
+ }
+ }
+
+ @Override
+ public void collect(int segDoc) throws IOException {
+ long ord = docValues.getOrd(segDoc);
+ if (ord != -1) {
+ long val = toGlobal.get(ord);
+ collectValFirstPhase(segDoc, val);
+ }
+ }
+ });
+
+ } else { // Numeric:
+
+ // TODO support SortedNumericDocValues
+ DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {
+ NumericDocValues values = null; //NN
+ Bits docsWithField = null; //NN
+
+ @Override public boolean needsScores() { return false; }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
+ setNextReaderFirstPhase(ctx);
+ values = DocValues.getNumeric(ctx.reader(), sf.getName());
+ docsWithField = DocValues.getDocsWithField(ctx.reader(), sf.getName());
+ }
+
+ @Override
+ public void collect(int segDoc) throws IOException {
+ long val = values.get(segDoc);
+ if (val != 0 || docsWithField.get(segDoc)) {
+ collectValFirstPhase(segDoc, val);
+ }
+ }
+ });
}
+ }
- ArrayList<SimpleOrderedMap> bucketList = new ArrayList<>(collectCount);
- res.add("buckets", bucketList);
-
- boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
-
- for (int slotNum : sortedSlots) {
- SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
- Comparable val = calc.bitsToValue(table.vals[slotNum]);
- bucket.add("val", val);
-
- Query filter = needFilter ? sf.getType().getFieldQuery(null, sf, calc.formatValue(val)) : null;
-
- fillBucket(bucket, table.counts[slotNum], slotNum, null, filter);
+ private void collectValFirstPhase(int segDoc, long val) throws IOException {
+ int slot = table.add(val); // this can trigger a rehash
- bucketList.add(bucket);
- }
+ // Our countAcc is virtual, so this is not needed:
+ // countAcc.incrementCount(slot, 1);
- return res;
+ super.collectFirstPhase(segDoc, slot);
}
private void doRehash(LongCounts table) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
index 37b9d9b..de1636e 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
@@ -32,7 +32,12 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-
+/**
+ * Accumulates statistics separated by a slot number.
+ * There is a separate statistic per slot. The slot is usually an ordinal into a set of values, e.g. tracking a count
+ * frequency <em>per term</em>.
+ * Sometimes there doesn't need to be a slot distinction, in which case there is just one nominal slot.
+ */
public abstract class SlotAcc implements Closeable {
String key; // todo...
protected final FacetContext fcontext;
@@ -210,9 +215,7 @@ abstract class DoubleFuncSlotAcc extends FuncSlotAcc {
@Override
public void reset() {
- for (int i=0; i<result.length; i++) {
- result[i] = initialValue;
- }
+ Arrays.fill(result, initialValue);
}
@Override
@@ -246,9 +249,7 @@ abstract class IntSlotAcc extends SlotAcc {
@Override
public void reset() {
- for (int i=0; i<result.length; i++) {
- result[i] = initialValue;
- }
+ Arrays.fill(result, initialValue);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b5df8a1/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 7b5a561..6ab25bb 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.tdunning.math.stats.AVLTreeDigest;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.hll.HLL;
@@ -43,12 +44,19 @@ public class TestJsonFacets extends SolrTestCaseHS {
private static SolrInstances servers; // for distributed testing
private static int origTableSize;
+ private static FacetField.FacetMethod origDefaultFacetMethod;
@BeforeClass
public static void beforeTests() throws Exception {
JSONTestUtil.failRepeatedKeys = true;
+
origTableSize = FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE;
FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing
+
+ origDefaultFacetMethod = FacetField.FacetMethod.DEFAULT_METHOD;
+ // instead of the following, see the constructor
+ //FacetField.FacetMethod.DEFAULT_METHOD = rand(FacetField.FacetMethod.values());
+
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@@ -62,12 +70,25 @@ public class TestJsonFacets extends SolrTestCaseHS {
public static void afterTests() throws Exception {
JSONTestUtil.failRepeatedKeys = false;
FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize;
+ FacetField.FacetMethod.DEFAULT_METHOD = origDefaultFacetMethod;
if (servers != null) {
servers.stop();
servers = null;
}
}
+ // tip: when debugging a test, comment out the @ParametersFactory and edit the constructor to be no-arg
+
+ @ParametersFactory
+ public static Iterable<Object[]> parameters() {
+ // wrap each enum val in an Object[] and return as Iterable
+ return () -> Arrays.stream(FacetField.FacetMethod.values()).map(it -> new Object[]{it}).iterator();
+ }
+
+ public TestJsonFacets(FacetField.FacetMethod defMethod) {
+ FacetField.FacetMethod.DEFAULT_METHOD = defMethod; // note: the real default is restored in afterTests
+ }
+
// attempt to reproduce https://github.com/Heliosearch/heliosearch/issues/33
@Test
public void testComplex() throws Exception {
@@ -180,8 +201,8 @@ public class TestJsonFacets extends SolrTestCaseHS {
client.commit();
}
-
- public void testStatsSimple() throws Exception {
+ @Test
+ public void testMethodStream() throws Exception {
Client client = Client.localClient();
indexSimple(client);
@@ -196,15 +217,15 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test streaming
assertJQ(req("q", "*:*", "rows", "0"
- , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream }}" +
- ", cat2:{terms:{field:'cat_s', method:stream, sort:'index asc' }}" + // default sort
- ", cat3:{terms:{field:'cat_s', method:stream, mincount:3 }}" + // mincount
- ", cat4:{terms:{field:'cat_s', method:stream, prefix:B }}" + // prefix
- ", cat5:{terms:{field:'cat_s', method:stream, offset:1 }}" + // offset
+ , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream }}" + // won't stream; need sort:index asc
+ ", cat2:{terms:{field:'cat_s', method:stream, sort:'index asc' }}" +
+ ", cat3:{terms:{field:'cat_s', method:stream, sort:'index asc', mincount:3 }}" + // mincount
+ ", cat4:{terms:{field:'cat_s', method:stream, sort:'index asc', prefix:B }}" + // prefix
+ ", cat5:{terms:{field:'cat_s', method:stream, sort:'index asc', offset:1 }}" + // offset
" }"
)
, "facets=={count:6 " +
- ", cat :{buckets:[{val:A, count:2},{val:B, count:3}]}" +
+ ", cat :{buckets:[{val:B, count:3},{val:A, count:2}]}" +
", cat2:{buckets:[{val:A, count:2},{val:B, count:3}]}" +
", cat3:{buckets:[{val:B, count:3}]}" +
", cat4:{buckets:[{val:B, count:3}]}" +
@@ -215,7 +236,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming under non-streaming
assertJQ(req("q", "*:*", "rows", "0"
- , "json.facet", "{ cat:{terms:{field:'cat_s', sort:'index asc', facet:{where:{terms:{field:where_s,method:stream}}} }}}"
+ , "json.facet", "{ cat:{terms:{field:'cat_s', sort:'index asc', facet:{where:{terms:{field:where_s,method:stream,sort:'index asc'}}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1},{val:NY,count:1}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2},{val:NY,count:1}]} }]}"
@@ -224,7 +245,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming under streaming
assertJQ(req("q", "*:*", "rows", "0"
- , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{where:{terms:{field:where_s,method:stream}}} }}}"
+ , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{where:{terms:{field:where_s,method:stream,sort:'index asc'}}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1},{val:NY,count:1}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2},{val:NY,count:1}]} }]}"
@@ -233,7 +254,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming with stats under streaming
assertJQ(req("q", "*:*", "rows", "0"
- , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{ where:{terms:{field:where_s,method:stream, facet:{x:'max(num_d)'} }}} }}}"
+ , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{ where:{terms:{field:where_s,method:stream,sort:'index asc',sort:'index asc', facet:{x:'max(num_d)'} }}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1,x:2.0},{val:NY,count:1,x:4.0}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2,x:11.0},{val:NY,count:1,x:-5.0}]} }]}"
@@ -243,7 +264,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming with stats under streaming with stats
assertJQ(req("q", "*:*", "rows", "0",
"facet","true"
- , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{ y:'min(num_d)', where:{terms:{field:where_s,method:stream, facet:{x:'max(num_d)'} }}} }}}"
+ , "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{ y:'min(num_d)', where:{terms:{field:where_s,method:stream,sort:'index asc', facet:{x:'max(num_d)'} }}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, y:2.0, where:{buckets:[{val:NJ,count:1,x:2.0},{val:NY,count:1,x:4.0}]} },{val:B, count:3, y:-9.0, where:{buckets:[{val:NJ,count:2,x:11.0},{val:NY,count:1,x:-5.0}]} }]}"
@@ -294,7 +315,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
}
@Test
- public void testDistrib() throws Exception {
+ public void testStatsDistrib() throws Exception {
initServers();
Client client = servers.getClient(random().nextInt());
client.queryDefaults().set( "shards", servers.getShards(), "debugQuery", Boolean.toString(random().nextBoolean()) );