You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2014/11/06 21:15:54 UTC
svn commit: r1637204 [1/2] - in /lucene/dev/branches/branch_5x: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/
...
Author: hossman
Date: Thu Nov 6 20:15:52 2014
New Revision: 1637204
URL: http://svn.apache.org/r1637204
Log:
SOLR-6351: Stats can now be nested under pivot values by adding a 'stats' local param (merge r1636772)
Added:
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java
- copied unchanged from r1636772, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java
- copied unchanged from r1636772, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java
- copied unchanged from r1636772, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/dev-tools/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_5x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_5x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_5x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_5x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_5x/lucene/README.txt (props changed)
lucene/dev/branches/branch_5x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene47WordDelimiterFilter.java (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/ASCIITLD.jflex-macro (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/SUPPLEMENTARY.jflex-macro (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/StandardTokenizerImpl40.java (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/StandardTokenizerImpl40.jflex (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.jflex (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/package.html (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLucene47WordDelimiterFilter.java (props changed)
lucene/dev/branches/branch_5x/lucene/backward-codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_5x/lucene/build.xml (props changed)
lucene/dev/branches/branch_5x/lucene/classification/ (props changed)
lucene/dev/branches/branch_5x/lucene/classification/build.xml (props changed)
lucene/dev/branches/branch_5x/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/branch_5x/lucene/classification/src/ (props changed)
lucene/dev/branches/branch_5x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_5x/lucene/demo/ (props changed)
lucene/dev/branches/branch_5x/lucene/expressions/ (props changed)
lucene/dev/branches/branch_5x/lucene/facet/ (props changed)
lucene/dev/branches/branch_5x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_5x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_5x/lucene/ivy-ignore-conflicts.properties (props changed)
lucene/dev/branches/branch_5x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_5x/lucene/ivy-versions.properties (props changed)
lucene/dev/branches/branch_5x/lucene/join/ (props changed)
lucene/dev/branches/branch_5x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_5x/lucene/memory/ (props changed)
lucene/dev/branches/branch_5x/lucene/misc/ (props changed)
lucene/dev/branches/branch_5x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_5x/lucene/queries/ (props changed)
lucene/dev/branches/branch_5x/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (props changed)
lucene/dev/branches/branch_5x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_5x/lucene/replicator/ (props changed)
lucene/dev/branches/branch_5x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/site/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/test-files/data/simple-bbox.txt (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/test-files/simple-Queries-BBox.txt (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/ (props changed)
lucene/dev/branches/branch_5x/lucene/tools/ (props changed)
lucene/dev/branches/branch_5x/lucene/version.properties (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_5x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_5x/solr/README.txt (props changed)
lucene/dev/branches/branch_5x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_5x/solr/bin/ (props changed)
lucene/dev/branches/branch_5x/solr/build.xml (props changed)
lucene/dev/branches/branch_5x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_5x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_5x/solr/contrib/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/request/DocValuesStats.java (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/SchemaManager.java (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/PivotListEntry.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java
lucene/dev/branches/branch_5x/solr/example/ (props changed)
lucene/dev/branches/branch_5x/solr/licenses/ (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_5x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_5x/solr/scripts/ (props changed)
lucene/dev/branches/branch_5x/solr/server/ (props changed)
lucene/dev/branches/branch_5x/solr/site/ (props changed)
lucene/dev/branches/branch_5x/solr/solrj/ (props changed)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
lucene/dev/branches/branch_5x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/branches/branch_5x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_5x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Thu Nov 6 20:15:52 2014
@@ -174,6 +174,10 @@ New Features
* SOLR-6670: change BALANCESLICEUNIQUE to BALANCESHARDUNIQUE. Also, the parameter
for ADDREPLICAPROP that used to be sliceUnique is now shardUnique. (Erick Erickson)
+* SOLR-6351: Stats can now be nested under pivot values by adding a 'stats' local param to
+ facet.pivot which refers to a 'tag' local param in one or more stats.field params.
+ (hossman, Vitaliy Zhovtyuk)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java Thu Nov 6 20:15:52 2014
@@ -18,15 +18,21 @@
package org.apache.solr.handler.component;
import org.apache.solr.util.PivotListEntry;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Collections;
+import java.util.Map;
+import java.util.Map.Entry;
public class PivotFacetHelper {
@@ -91,31 +97,63 @@ public class PivotFacetHelper {
/** @see PivotListEntry#VALUE */
public static Comparable getValue(NamedList<Object> pivotList) {
- return (Comparable) PivotFacetHelper.retrieve(PivotListEntry.VALUE,
- pivotList);
+ return (Comparable) PivotListEntry.VALUE.extract(pivotList);
}
/** @see PivotListEntry#FIELD */
public static String getField(NamedList<Object> pivotList) {
- return (String) PivotFacetHelper.retrieve(PivotListEntry.FIELD, pivotList);
+ return (String) PivotListEntry.FIELD.extract(pivotList);
}
/** @see PivotListEntry#COUNT */
public static Integer getCount(NamedList<Object> pivotList) {
- return (Integer) PivotFacetHelper.retrieve(PivotListEntry.COUNT, pivotList);
+ return (Integer) PivotListEntry.COUNT.extract(pivotList);
}
/** @see PivotListEntry#PIVOT */
public static List<NamedList<Object>> getPivots(NamedList<Object> pivotList) {
- int pivotIdx = pivotList.indexOf(PivotListEntry.PIVOT.getName(), 0);
- if (pivotIdx > -1) {
- return (List<NamedList<Object>>) pivotList.getVal(pivotIdx);
- }
- return null;
+ return (List<NamedList<Object>>) PivotListEntry.PIVOT.extract(pivotList);
}
- private static Object retrieve(PivotListEntry entryToGet, NamedList<Object> pivotList) {
- return pivotList.get(entryToGet.getName(), entryToGet.getIndex());
+ /** @see PivotListEntry#STATS */
+ public static NamedList<NamedList<NamedList<?>>> getStats(NamedList<Object> pivotList) {
+ return (NamedList<NamedList<NamedList<?>>>) PivotListEntry.STATS.extract(pivotList);
+ }
+
+ /**
+ * Given a mapping of keys to {@link StatsValues} representing the currently
+ * known "merged" stats (which may be null if none exist yet), and a
+ * {@link NamedList} containing the "stats" response block returned by an individual
+ * shard, this method accumulates the stasts for each {@link StatsField} found in
+ * the shard response with the existing mergeStats
+ *
+ * @return the original <code>merged</code> Map after modifying, or a new Map if the <code>merged</code> param was originally null.
+ * @see StatsInfo#getStatsField
+ * @see StatsValuesFactory#createStatsValues
+ * @see StatsValues#accumulate(NamedList)
+ */
+ public static Map<String,StatsValues> mergeStats
+ (Map<String,StatsValues> merged,
+ NamedList<NamedList<NamedList<?>>> remoteWrapper,
+ StatsInfo statsInfo) {
+
+ if (null == merged) merged = new LinkedHashMap<String,StatsValues>();
+
+ NamedList<NamedList<?>> remoteStats = StatsComponent.unwrapStats(remoteWrapper);
+
+ for (Entry<String,NamedList<?>> entry : remoteStats) {
+ StatsValues receivingStatsValues = merged.get(entry.getKey());
+ if (receivingStatsValues == null) {
+ StatsField recievingStatsField = statsInfo.getStatsField(entry.getKey());
+ if (null == recievingStatsField) {
+ throw new SolrException(ErrorCode.SERVER_ERROR , "No stats.field found corrisponding to pivot stats recieved from shard: "+entry.getKey());
+ }
+ receivingStatsValues = StatsValuesFactory.createStatsValues(recievingStatsField);
+ merged.put(entry.getKey(), receivingStatsValues);
+ }
+ receivingStatsValues.accumulate(entry.getValue());
+ }
+ return merged;
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java Thu Nov 6 20:15:52 2014
@@ -23,20 +23,26 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SyntaxError;
+import org.apache.solr.util.PivotListEntry;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.common.params.StatsParams;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.lucene.search.Query;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -63,9 +69,15 @@ public class PivotFacetProcessor extends
if (!rb.doFacets || pivots == null)
return null;
+ // rb._statsInfo may be null if stats=false, ie: refine requests
+ // if that's the case, but we need to refine w/stats, then we'll lazy init our
+ // own instance of StatsInfo
+ StatsInfo statsInfo = rb._statsInfo;
+
SimpleOrderedMap<List<NamedList<Object>>> pivotResponse = new SimpleOrderedMap<>();
for (String pivotList : pivots) {
try {
+ // NOTE: this sets localParams (SimpleFacets is stateful)
this.parseParams(FacetParams.FACET_PIVOT, pivotList);
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST, e);
@@ -84,15 +96,37 @@ public class PivotFacetProcessor extends
}
}
- //REFINEMENT
- String fieldValueKey = localParams == null ? null : localParams.get(PivotFacet.REFINE_PARAM);
- if(fieldValueKey != null ){
- String[] refinementValuesByField = params.getParams(PivotFacet.REFINE_PARAM+fieldValueKey);
+ // start by assuing no local params...
+
+ String refineKey = null; // no local => no refinement
+ List<StatsField> statsFields = Collections.emptyList(); // no local => no stats
+
+ if (null != localParams) {
+ // we might be refining..
+ refineKey = localParams.get(PivotFacet.REFINE_PARAM);
+
+ String statsLocalParam = localParams.get(StatsParams.STATS);
+ if (null != refineKey
+ && null != statsLocalParam
+ && null == statsInfo) {
+ // we are refining and need to compute stats,
+ // but stats component hasn't inited StatsInfo (because we
+ // don't need/want top level stats when refining) so we lazy init
+ // our own copy of StatsInfo
+ statsInfo = new StatsInfo(rb);
+ }
+ statsFields = getTaggedStatsFields(statsInfo, statsLocalParam);
+ }
+
+ if (null != refineKey) {
+ String[] refinementValuesByField
+ = params.getParams(PivotFacet.REFINE_PARAM + refineKey);
+
for(String refinements : refinementValuesByField){
- pivotResponse.addAll(processSingle(pivotFields, refinements));
+ pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields));
}
} else{
- pivotResponse.addAll(processSingle(pivotFields, null));
+ pivotResponse.addAll(processSingle(pivotFields, null, statsFields));
}
}
return pivotResponse;
@@ -102,9 +136,13 @@ public class PivotFacetProcessor extends
* Process a single branch of refinement values for a specific pivot
* @param pivotFields the ordered list of fields in this pivot
* @param refinements the comma seperate list of refinement values corrisponding to each field in the pivot, or null if there are no refinements
+ * @param statsFields List of {@link StatsField} instances to compute for each pivot value
*/
- private SimpleOrderedMap<List<NamedList<Object>>> processSingle(List<String> pivotFields,
- String refinements) throws IOException {
+ private SimpleOrderedMap<List<NamedList<Object>>> processSingle
+ (List<String> pivotFields,
+ String refinements,
+ List<StatsField> statsFields) throws IOException {
+
SolrIndexSearcher searcher = rb.req.getSearcher();
SimpleOrderedMap<List<NamedList<Object>>> pivotResponse = new SimpleOrderedMap<>();
@@ -141,18 +179,54 @@ public class PivotFacetProcessor extends
if(pivotFields.size() > 1) {
String subField = pivotFields.get(1);
pivotResponse.add(key,
- doPivots(facetCounts, field, subField, fnames, vnames, this.docs));
+ doPivots(facetCounts, field, subField, fnames, vnames, this.docs, statsFields));
} else {
- pivotResponse.add(key, doPivots(facetCounts, field, null, fnames, vnames, this.docs));
+ pivotResponse.add(key, doPivots(facetCounts, field, null, fnames, vnames, this.docs, statsFields));
}
return pivotResponse;
}
/**
+ * returns the {@link StatsField} instances that should be computed for a pivot
+ * based on the 'stats' local params used.
+ *
+ * @return A list of StatsFields to comput for this pivot, or the empty list if none
+ */
+ private static List<StatsField> getTaggedStatsFields(StatsInfo statsInfo,
+ String statsLocalParam) {
+ if (null == statsLocalParam || null == statsInfo) {
+ return Collections.emptyList();
+ }
+
+ List<StatsField> fields = new ArrayList<>(7);
+ List<String> statsAr = StrUtils.splitSmart(statsLocalParam, ',');
+
+ // TODO: for now, we only support a single tag name - we reserve using
+ // ',' as a possible delimeter for logic related to only computing stats
+ // at certain levels -- see SOLR-6663
+ if (1 < statsAr.size()) {
+ String msg = StatsParams.STATS + " local param of " + FacetParams.FACET_PIVOT +
+ "may not include tags separated by a comma - please use a common tag on all " +
+ StatsParams.STATS_FIELD + " params you wish to compute under this pivot";
+ throw new SolrException(ErrorCode.BAD_REQUEST, msg);
+ }
+
+ for(String stat : statsAr) {
+ fields.addAll(statsInfo.getStatsFieldsByTag(stat));
+ }
+ return fields;
+ }
+
+ /**
* Recursive function to compute all the pivot counts for the values under teh specified field
*/
protected List<NamedList<Object>> doPivots(NamedList<Integer> superFacets,
- String field, String subField, Deque<String> fnames,Deque<String> vnames,DocSet docs) throws IOException {
+ String field, String subField,
+ Deque<String> fnames, Deque<String> vnames,
+ DocSet docs, List<StatsField> statsFields)
+ throws IOException {
+
+ boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
SolrIndexSearcher searcher = rb.req.getSearcher();
// TODO: optimize to avoid converting to an external string and then having to convert back to internal below
@@ -169,6 +243,7 @@ public class PivotFacetProcessor extends
// Only sub-facet if parent facet has positive count - still may not be any values for the sub-field though
if (kv.getValue() >= getMinCountForField(field)) {
final String fieldValue = kv.getKey();
+ final int pivotCount = kv.getValue();
SimpleOrderedMap<Object> pivot = new SimpleOrderedMap<>();
pivot.add( "field", field );
@@ -178,7 +253,7 @@ public class PivotFacetProcessor extends
ftype.readableToIndexed(fieldValue, termval);
pivot.add( "value", ftype.toObject(sfield, termval.get()) );
}
- pivot.add( "count", kv.getValue() );
+ pivot.add( "count", pivotCount );
DocSet subset = getSubset(docs, sfield, fieldValue);
@@ -195,8 +270,16 @@ public class PivotFacetProcessor extends
}
if (facetCounts.size() >= 1) {
- pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, subset) );
+ pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, subset, statsFields ) );
+ }
+ }
+ if ((isShard || 0 < pivotCount) && ! statsFields.isEmpty()) {
+ Map<String, StatsValues> stv = new LinkedHashMap<>();
+ for (StatsField statsField : statsFields) {
+ stv.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(subset));
}
+ // for pivots, we *always* include requested stats - even if 'empty'
+ pivot.add("stats", StatsComponent.convertToResponse(true, stv));
}
values.add( pivot );
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java Thu Nov 6 20:15:52 2014
@@ -21,11 +21,13 @@ import java.util.BitSet;
import java.util.Date;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.TrieDateField;
+import org.apache.solr.search.QueryParsing;
import org.apache.solr.util.PivotListEntry;
/**
@@ -45,6 +47,7 @@ public class PivotFacetValue {
// child can't be final, circular ref on construction
private PivotFacetField childPivot = null;
private int count; // mutable
+ private Map<String, StatsValues> statsValues = null;
private PivotFacetValue(PivotFacetField parent, Comparable val) {
this.parentPivot = parent;
@@ -114,6 +117,7 @@ public class PivotFacetValue {
Comparable pivotVal = null;
int pivotCount = 0;
List<NamedList<Object>> childPivotData = null;
+ NamedList<NamedList<NamedList<?>>> statsValues = null;
for (int i = 0; i < pivotData.size(); i++) {
String key = pivotData.getName(i);
@@ -135,6 +139,9 @@ public class PivotFacetValue {
case PIVOT:
childPivotData = (List<NamedList<Object>>)value;
break;
+ case STATS:
+ statsValues = (NamedList<NamedList<NamedList<?>>>) value;
+ break;
default:
throw new RuntimeException("PivotListEntry contains unaccounted for item: " + entry);
}
@@ -143,6 +150,9 @@ public class PivotFacetValue {
PivotFacetValue newPivotFacet = new PivotFacetValue(parentField, pivotVal);
newPivotFacet.count = pivotCount;
newPivotFacet.sourceShards.set(shardNumber);
+ if(statsValues != null) {
+ newPivotFacet.statsValues = PivotFacetHelper.mergeStats(null, statsValues, rb._statsInfo);
+ }
newPivotFacet.childPivot = PivotFacetField.createFromListOfNamedLists(shardNumber, rb, newPivotFacet, childPivotData);
@@ -171,6 +181,11 @@ public class PivotFacetValue {
if (childPivot != null && childPivot.convertToListOfNamedLists() != null) {
newList.add(PivotListEntry.PIVOT.getName(), childPivot.convertToListOfNamedLists());
}
+ if (null != statsValues) {
+ newList.add(PivotListEntry.STATS.getName(),
+ // for pivots, we *always* include requested stats - even if 'empty'
+ StatsComponent.convertToResponse(true, statsValues));
+ }
return newList;
}
@@ -187,6 +202,10 @@ public class PivotFacetValue {
if (!shardHasContributed(shardNumber)) {
sourceShards.set(shardNumber);
count += PivotFacetHelper.getCount(value);
+ NamedList<NamedList<NamedList<?>>> stats = PivotFacetHelper.getStats(value);
+ if (stats != null) {
+ statsValues = PivotFacetHelper.mergeStats(statsValues, stats, rb._statsInfo);
+ }
}
List<NamedList<Object>> shardChildPivots = PivotFacetHelper.getPivots(value);
@@ -197,7 +216,7 @@ public class PivotFacetValue {
childPivot.contributeFromShard(shardNumber, rb, shardChildPivots);
}
}
-
+
public String toString(){
return String.format(Locale.ROOT, "F:%s V:%s Co:%d Ch?:%s",
parentPivot.field, value, count, (this.childPivot !=null));
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Thu Nov 6 20:15:52 2014
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
@@ -56,22 +54,14 @@ public class StatsComponent extends Sear
if (!rb.doStats) return;
boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
- NamedList<Object> out = new SimpleOrderedMap<>();
- NamedList<Object> stats_fields = new SimpleOrderedMap<>();
+ Map<String, StatsValues> statsValues = new LinkedHashMap<>();
for (StatsField statsField : rb._statsInfo.getStatsFields()) {
DocSet docs = statsField.computeBaseDocSet();
- NamedList<?> stv = statsField.computeLocalStatsValues(docs).getStatsValues();
-
- if (isShard == true || (Long) stv.get("count") > 0) {
- stats_fields.add(statsField.getOutputKey(), stv);
- } else {
- stats_fields.add(statsField.getOutputKey(), null);
- }
+ statsValues.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(docs));
}
- out.add("stats_fields", stats_fields);
- rb.rsp.add( "stats", out );
+ rb.rsp.add( "stats", convertToResponse(isShard, statsValues) );
}
@Override
@@ -86,6 +76,8 @@ public class StatsComponent extends Sear
if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
sreq.purpose |= ShardRequest.PURPOSE_GET_STATS;
} else {
+
+
// turn off stats on other requests
sreq.params.set(StatsParams.STATS, "false");
// we could optionally remove stats params
@@ -101,7 +93,8 @@ public class StatsComponent extends Sear
for (ShardResponse srsp : sreq.responses) {
NamedList stats = null;
try {
- stats = (NamedList) srsp.getSolrResponse().getResponse().get("stats");
+ stats = (NamedList<NamedList<NamedList<?>>>)
+ srsp.getSolrResponse().getResponse().get("stats");
} catch (Exception e) {
if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false)) {
continue; // looks like a shard did not return anything
@@ -110,7 +103,7 @@ public class StatsComponent extends Sear
"Unable to read stats info for shard: " + srsp.getShard(), e);
}
- NamedList stats_fields = (NamedList) stats.get("stats_fields");
+ NamedList stats_fields = unwrapStats(stats);
if (stats_fields != null) {
for (int i = 0; i < stats_fields.size(); i++) {
String key = stats_fields.getName(i);
@@ -129,26 +122,44 @@ public class StatsComponent extends Sear
// so that "result" is already stored in the response (for aesthetics)
Map<String, StatsValues> allStatsValues = rb._statsInfo.getAggregateStatsValues();
+ rb.rsp.add("stats", convertToResponse(false, allStatsValues));
+
+ rb._statsInfo = null; // free some objects
+ }
+
+ /**
+ * Helper to pull the "stats_fields" out of the extra "stats" wrapper
+ */
+ public static NamedList<NamedList<?>> unwrapStats(NamedList<NamedList<NamedList<?>>> stats) {
+ if (null == stats) return null;
+
+ return stats.get("stats_fields");
+ }
- NamedList<NamedList<Object>> stats = new SimpleOrderedMap<>();
- NamedList<Object> stats_fields = new SimpleOrderedMap<>();
+ /**
+ * Given a map of {@link StatsValues} using the appropriate response key,
+ * builds up the neccessary "stats" data structure for including in the response --
+ * including the esoteric "stats_fields" wrapper.
+ */
+ public static NamedList<NamedList<NamedList<?>>> convertToResponse
+ (boolean force, Map<String,StatsValues> statsValues) {
+
+ NamedList<NamedList<NamedList<?>>> stats = new SimpleOrderedMap<>();
+ NamedList<NamedList<?>> stats_fields = new SimpleOrderedMap<>();
stats.add("stats_fields", stats_fields);
- for (Map.Entry<String,StatsValues> entry : allStatsValues.entrySet()) {
+ for (Map.Entry<String,StatsValues> entry : statsValues.entrySet()) {
String key = entry.getKey();
NamedList stv = entry.getValue().getStatsValues();
- if ((Long) stv.get("count") != 0) {
+ if (force || ((Long) stv.get("count") != 0)) {
stats_fields.add(key, stv);
} else {
stats_fields.add(key, null);
}
}
-
- rb.rsp.add("stats", stats);
- rb._statsInfo = null; // free some objects
+ return stats;
}
-
/////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
@@ -168,6 +179,8 @@ class StatsInfo {
private final ResponseBuilder rb;
private final List<StatsField> statsFields = new ArrayList<>(7);
private final Map<String, StatsValues> distribStatsValues = new LinkedHashMap<>();
+ private final Map<String, StatsField> statsFieldMap = new LinkedHashMap<>();
+ private final Map<String, List<StatsField>> tagToStatsFields = new LinkedHashMap<>();
public StatsInfo(ResponseBuilder rb) {
this.rb = rb;
@@ -177,10 +190,19 @@ class StatsInfo {
// no stats.field params, nothing to parse.
return;
}
-
+
for (String paramValue : statsParams) {
StatsField current = new StatsField(rb, paramValue);
statsFields.add(current);
+ for (String tag : current.getTagList()) {
+ List<StatsField> fieldList = tagToStatsFields.get(tag);
+ if (fieldList == null) {
+ fieldList = new ArrayList<>();
+ }
+ fieldList.add(current);
+ tagToStatsFields.put(tag, fieldList);
+ }
+ statsFieldMap.put(current.getOutputKey(), current);
distribStatsValues.put(current.getOutputKey(),
StatsValuesFactory.createStatsValues(current));
}
@@ -192,7 +214,31 @@ class StatsInfo {
* as part of this request
*/
public List<StatsField> getStatsFields() {
- return Collections.<StatsField>unmodifiableList(statsFields);
+ return Collections.unmodifiableList(statsFields);
+ }
+
+ /**
+ * Returns the {@link StatsField} associated with the specified (effective)
+ * outputKey, or null if there was no {@link StatsParams#STATS_FIELD} param
+ * that would corrispond with that key.
+ */
+ public StatsField getStatsField(String outputKey) {
+ return statsFieldMap.get(outputKey);
+ }
+
+ /**
+ * Return immutable list of {@link StatsField} instances by string tag local parameter.
+ *
+ * @param tag tag local parameter
+ * @return list of stats fields
+ */
+ public List<StatsField> getStatsFieldsByTag(String tag) {
+ List<StatsField> raw = tagToStatsFields.get(tag);
+ if (null == raw) {
+ return Collections.emptyList();
+ } else {
+ return Collections.unmodifiableList(raw);
+ }
}
/**
@@ -203,7 +249,7 @@ class StatsInfo {
* will never be null.
*/
public Map<String, StatsValues> getAggregateStatsValues() {
- return Collections.<String, StatsValues>unmodifiableMap(distribStatsValues);
+ return Collections.unmodifiableMap(distribStatsValues);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsField.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsField.java Thu Nov 6 20:15:52 2014
@@ -70,6 +70,7 @@ public class StatsField {
private final String key;
private final boolean calcDistinct; // TODO: put this inside localParams ? SOLR-6349 ?
private final String[] facets;
+ private final List<String> tagList;
private final List<String> excludeTagList;
/**
@@ -147,6 +148,10 @@ public class StatsField {
String[] facets = params.getFieldParams(key, StatsParams.STATS_FACET);
this.facets = (null == facets) ? new String[0] : facets;
+ String tagStr = localParams.get(CommonParams.TAG);
+ this.tagList = (null == tagStr)
+ ? Collections.<String>emptyList()
+ : StrUtils.splitSmart(tagStr,',');
// figure out if we need a special base DocSet
String excludeStr = localParams.get(CommonParams.EXCLUDE);
@@ -363,6 +368,11 @@ public class StatsField {
return calcDistinct;
}
+
+ public List<String> getTagList() {
+ return tagList;
+ }
+
public String toString() {
return "StatsField<" + originalParam + ">";
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java Thu Nov 6 20:15:52 2014
@@ -316,8 +316,6 @@ class NumericStatsValues extends Abstrac
public NumericStatsValues(StatsField statsField) {
super(statsField);
- min = Double.POSITIVE_INFINITY;
- max = Double.NEGATIVE_INFINITY;
}
@Override
@@ -353,8 +351,22 @@ class NumericStatsValues extends Abstrac
*/
@Override
protected void updateMinMax(Number min, Number max) {
- this.min = Math.min(this.min.doubleValue(), min.doubleValue());
- this.max = Math.max(this.max.doubleValue(), max.doubleValue());
+ if (null == min) {
+ assert null == max : "min is null but max isn't ? ==> " + max;
+ return; // No-Op
+ }
+
+ assert null != max : "max is null but min isn't ? ==> " + min;
+
+ // we always use the double value, because that way the response Object class is
+ // consistent regardless of wether we only have 1 value or many that we min/max
+ //
+ // TODO: would be nice to have subclasses for each type of Number ... breaks backcompat
+ double minD = min.doubleValue();
+ double maxD = max.doubleValue();
+
+ this.min = (null == this.min) ? minD : Math.min(this.min.doubleValue(), minD);
+ this.max = (null == this.max) ? maxD : Math.max(this.max.doubleValue(), maxD);
}
/**
@@ -594,7 +606,7 @@ class StringStatsValues extends Abstract
// Add no statistics
}
- /**
+ /**
* Determines which of the given Strings is the maximum, as computed by {@link String#compareTo(String)}
*
* @param str1 String to compare against b
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/PivotListEntry.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/PivotListEntry.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/PivotListEntry.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/PivotListEntry.java Thu Nov 6 20:15:52 2014
@@ -17,6 +17,10 @@ package org.apache.solr.util;
* limitations under the License.
*/
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
+
import java.util.Locale;
/**
@@ -24,16 +28,28 @@ import java.util.Locale;
*/
public enum PivotListEntry {
- FIELD(0),
+ // mandatory entries with exact indexes
+ FIELD(0),
VALUE(1),
COUNT(2),
- PIVOT(3);
+ // optional entries
+ PIVOT,
+ STATS;
- // we could just use the ordinal(), but safer to be very explicit
- private final int index;
+ private static final int MIN_INDEX_OF_OPTIONAL = 3;
+
+ /**
+ * Given a NamedList representing a Pivot Value, this is Minimum Index at
+ * which this PivotListEntry may exist
+ */
+ private final int minIndex;
- private PivotListEntry(int index) {
- this.index = index;
+ private PivotListEntry() {
+ this.minIndex = MIN_INDEX_OF_OPTIONAL;
+ }
+ private PivotListEntry(int minIndex) {
+ assert minIndex < MIN_INDEX_OF_OPTIONAL;
+ this.minIndex = minIndex;
}
/**
@@ -53,10 +69,19 @@ public enum PivotListEntry {
}
/**
- * Indec of this entry when used in response
+ * Given a {@link NamedList} representing a Pivot Value, extracts the Object
+ * which corrisponds to this {@link PivotListEntry}, or returns null if not found.
*/
- public int getIndex() {
- return index;
+ public Object extract(NamedList<Object> pivotList) {
+ if (this.minIndex < MIN_INDEX_OF_OPTIONAL) {
+ // a mandatory entry at an exact index.
+ assert this.getName().equals(pivotList.getName(this.minIndex));
+ assert this.minIndex < pivotList.size();
+ return pivotList.getVal(this.minIndex);
+ }
+ // otherweise...
+ // scan starting at the min/optional index
+ return pivotList.get(this.getName(), this.minIndex);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java Thu Nov 6 20:15:52 2014
@@ -16,17 +16,22 @@
*/
package org.apache.solr.cloud;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.schema.TrieDateField;
+import org.apache.solr.common.params.FacetParams; // jdoc lint
import static org.apache.solr.common.params.FacetParams.*;
import org.apache.commons.lang.StringUtils;
@@ -92,6 +97,9 @@ public class TestCloudPivotFacet extends
@Override
public void doTest() throws Exception {
+
+ sanityCheckAssertDoubles();
+
waitForThingsToLevelOut(30000); // TODO: why whould we have to wait?
//
handle.clear();
@@ -107,7 +115,7 @@ public class TestCloudPivotFacet extends
for (int i = 1; i <= numDocs; i++) {
SolrInputDocument doc = buildRandomDocument(i);
- // not efficient, but it garuntees that even if people change buildRandomDocument
+ // not efficient, but it guarantees that even if people change buildRandomDocument
// we'll always have the full list of fields w/o needing to keep code in sync
fieldNameSet.addAll(doc.getFieldNames());
@@ -119,7 +127,7 @@ public class TestCloudPivotFacet extends
assertTrue("WTF, bogus field exists?", fieldNameSet.add("bogus_not_in_any_doc_s"));
final String[] fieldNames = fieldNameSet.toArray(new String[fieldNameSet.size()]);
- Arrays.sort(fieldNames); // need determinism for buildRandomPivot calls
+ Arrays.sort(fieldNames); // need determinism when picking random fields
for (int i = 0; i < 5; i++) {
@@ -134,10 +142,28 @@ public class TestCloudPivotFacet extends
baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(),200,numDocs) + "]");
}
- ModifiableSolrParams pivotP = params(FACET,"true",
- FACET_PIVOT, buildRandomPivot(fieldNames));
+ final boolean stats = random().nextBoolean();
+ if (stats) {
+ baseP.add(StatsParams.STATS, "true");
+
+ // if we are doing stats, then always generated the same # of STATS_FIELD
+ // params, using multiple tags from a fixed set, but with diff fieldName values.
+ // later, each pivot will randomly pick a tag.
+ baseP.add(StatsParams.STATS_FIELD, "{!key=sk1 tag=st1,st2}" +
+ pickRandomStatsFields(fieldNames));
+ baseP.add(StatsParams.STATS_FIELD, "{!key=sk2 tag=st2,st3}" +
+ pickRandomStatsFields(fieldNames));
+ baseP.add(StatsParams.STATS_FIELD, "{!key=sk3 tag=st3,st4}" +
+ pickRandomStatsFields(fieldNames));
+ // NOTE: there's a chance that some of those stats field names
+ // will be the same, but if so, all the better to test that edge case
+ }
+
+ ModifiableSolrParams pivotP = params(FACET,"true");
+ pivotP.add(FACET_PIVOT, buildPivotParamValue(buildRandomPivot(fieldNames)));
+
if (random().nextBoolean()) {
- pivotP.add(FACET_PIVOT, buildRandomPivot(fieldNames));
+ pivotP.add(FACET_PIVOT, buildPivotParamValue(buildRandomPivot(fieldNames)));
}
// keep limit low - lots of unique values, and lots of depth in pivots
@@ -268,7 +294,7 @@ public class TestCloudPivotFacet extends
params("fq", buildFilter(constraint)));
List<PivotField> subPivots = null;
try {
- assertNumFound(pivotName, constraint.getCount(), p);
+ assertPivotData(pivotName, constraint, p);
subPivots = constraint.getPivot();
} catch (Exception e) {
throw new RuntimeException(pivotName + ": count query failed: " + p + ": " +
@@ -286,6 +312,97 @@ public class TestCloudPivotFacet extends
}
/**
+ * Executes a query and compares the results with the data available in the
+ * {@link PivotField} constraint -- this method is not recursive, and doesn't
+ * check anything about the sub-pivots (if any).
+ *
+ * @param pivotName pivot name
+ * @param constraint filters on pivot
+ * @param params base solr parameters
+ */
+ private void assertPivotData(String pivotName, PivotField constraint, SolrParams params)
+ throws SolrServerException {
+
+ SolrParams p = SolrParams.wrapDefaults(params("rows","0"), params);
+ QueryResponse res = cloudClient.query(p);
+ String msg = pivotName + ": " + p;
+
+ assertNumFound(msg, constraint.getCount(), res);
+
+ if ( p.getBool(StatsParams.STATS, false) ) {
+ // only check stats if stats expected
+ assertPivotStats(msg, constraint, res);
+ }
+ }
+
+ /**
+ * Compare top level stats in response with stats from pivot constraint
+ */
+ private void assertPivotStats(String message, PivotField constraint, QueryResponse response) throws SolrServerException {
+
+ if (null == constraint.getFieldStatsInfo()) {
+ // no stats for this pivot, nothing to check
+
+ // TODO: use a trace param to know if/how-many to expect ?
+ log.info("No stats to check for => " + message);
+ return;
+ }
+
+ Map<String, FieldStatsInfo> actualFieldStatsInfoMap = response.getFieldStatsInfo();
+
+ for (FieldStatsInfo pivotStats : constraint.getFieldStatsInfo().values()) {
+ String statsKey = pivotStats.getName();
+
+ FieldStatsInfo actualStats = actualFieldStatsInfoMap.get(statsKey);
+
+ if (actualStats == null) {
+ // handle case for not found stats (using stats query)
+ //
+ // these has to be a special case check due to the legacy behavior of "top level"
+ // StatsComponent results being "null" (and not even included in the
+ // getFieldStatsInfo() Map due to specila SolrJ logic)
+
+ log.info("Requested stats missing in verification query, pivot stats: " + pivotStats);
+ assertEquals("Special Count", 0L, pivotStats.getCount().longValue());
+ assertEquals("Special Missing",
+ constraint.getCount(), pivotStats.getMissing().longValue());
+
+ } else {
+ // regular stats, compare everything...
+
+ assert actualStats != null;
+ String msg = " of " + statsKey + " => " + message;
+
+ assertEquals("Min" + msg, pivotStats.getMin(), actualStats.getMin());
+ assertEquals("Max" + msg, pivotStats.getMax(), actualStats.getMax());
+ assertEquals("Mean" + msg, pivotStats.getMean(), actualStats.getMean());
+ assertEquals("Sum" + msg, pivotStats.getSum(), actualStats.getSum());
+ assertEquals("Count" + msg, pivotStats.getCount(), actualStats.getCount());
+ assertEquals("Missing" + msg, pivotStats.getMissing(), actualStats.getMissing());
+
+ assertDoubles("Stddev" + msg, pivotStats.getStddev(), actualStats.getStddev());
+ assertDoubles("SumOfSquares" + msg,
+ pivotStats.getSumOfSquares(), actualStats.getSumOfSquares());
+ }
+ }
+
+ if (constraint.getFieldStatsInfo().containsKey("sk2")) { // cheeseball hack
+ // if "sk2" was one of hte stats we computed, then we must have also seen
+ // sk1 or sk3 because of the way the tags are fixed
+ assertEquals("had stats sk2, but not another stat?",
+ 2, constraint.getFieldStatsInfo().size());
+ } else {
+ // if we did not see "sk2", then 1 of the others must be alone
+ assertEquals("only expected 1 stat",
+ 1, constraint.getFieldStatsInfo().size());
+ assertTrue("not sk1 or sk3",
+ constraint.getFieldStatsInfo().containsKey("sk1") ||
+ constraint.getFieldStatsInfo().containsKey("sk3"));
+ }
+
+ }
+
+ /**
* Verify that the PivotFields we're lookin at doesn't violate any of the expected
* behaviors based on the <code>TRACE_*</code> params found in the base params
*/
@@ -364,6 +481,39 @@ public class TestCloudPivotFacet extends
return StringUtils.join(fields, ",");
}
+ /**
+ * Picks a random field to use for Stats
+ */
+ private static String pickRandomStatsFields(String[] fieldNames) {
+ // we need to skip boolean fields when computing stats
+ String fieldName;
+ do {
+ fieldName = fieldNames[TestUtil.nextInt(random(),0,fieldNames.length-1)];
+ }
+ while(fieldName.endsWith("_b") || fieldName.endsWith("_b1")) ;
+
+ return fieldName;
+ }
+
+ /**
+ * Generates a random {@link FacetParams#FACET_PIVOT} value w/ local params
+ * using the specified pivotValue.
+ */
+ private static String buildPivotParamValue(String pivotValue) {
+ // randomly decide which stat tag to use
+
+ // if this is 0, or stats aren't enabled, we'll be asking for a tag that doesn't exist
+ // ...which should be fine (just like excluding a taged fq that doesn't exist)
+ final int statTag = TestUtil.nextInt(random(), -1, 4);
+
+ if (0 <= statTag) {
+ // only use 1 tag name in the 'stats' localparam - see SOLR-6663
+ return "{!stats=st"+statTag+"}" + pivotValue;
+ } else {
+ // statTag < 0 == sanity check the case of a pivot w/o any stats
+ return pivotValue;
+ }
+ }
/**
* Creates a document with randomized field values, some of which be missing values,
@@ -512,16 +662,80 @@ public class TestCloudPivotFacet extends
}
/**
- * Asserts the number of docs matching the SolrParams aganst the cloudClient
+ * Asserts the number of docs found in the response
*/
- private void assertNumFound(String msg, int expected, SolrParams p)
+ private void assertNumFound(String msg, int expected, QueryResponse response)
throws SolrServerException {
countNumFoundChecks++;
- SolrParams params = SolrParams.wrapDefaults(params("rows","0"), p);
- assertEquals(msg + ": " + params,
- expected, cloudClient.query(params).getResults().getNumFound());
+ assertEquals(msg, expected, response.getResults().getNumFound());
+ }
+
+ /**
+ * Given two objects, asserts that they are either both null, or both Numbers
+ * with double values that are equally-ish with a "small" epsilon (relative to the
+ * scale of the expected value)
+ *
+ * @see Number#doubleValue
+ */
+ private void assertDoubles(String msg, Object expected, Object actual) {
+ if (null == expected || null == actual) {
+ assertEquals(msg, expected, actual);
+ } else {
+ assertTrue(msg + " ... expected not a double: " +
+ expected + "=>" + expected.getClass(),
+ expected instanceof Number);
+ assertTrue(msg + " ... actual not a double: " +
+ actual + "=>" + actual.getClass(),
+ actual instanceof Number);
+
+ // compute an epsilon relative to the size of the expected value
+ double expect = ((Number)expected).doubleValue();
+ double epsilon = expect * 0.1E-7D;
+
+ assertEquals(msg, expect, ((Number)actual).doubleValue(), epsilon);
+
+ }
+ }
+
+ /**
+ * test the test
+ */
+ private void sanityCheckAssertDoubles() {
+ assertDoubles("Null?", null, null);
+ assertDoubles("big",
+ new Double(2.3005390038169265E9),
+ new Double(2.300539003816927E9));
+ assertDoubles("small",
+ new Double(2.3005390038169265E-9),
+ new Double(2.300539003816927E-9));
+ try {
+ assertDoubles("non-null", null, 42);
+ fail("expected was null");
+ } catch (AssertionError e) {}
+ try {
+ assertDoubles("non-null", 42, null);
+ fail("actual was null");
+ } catch (AssertionError e) {}
+ try {
+ assertDoubles("non-number", 42, "foo");
+ fail("actual was non-number");
+ } catch (AssertionError e) {}
+ try {
+ assertDoubles("diff",
+ new Double(2.3005390038169265E9),
+ new Double(2.267272520100462E9));
+ fail("big & diff");
+ } catch (AssertionError e) {}
+ try {
+ assertDoubles("diff",
+ new Double(2.3005390038169265E-9),
+ new Double(2.267272520100462E-9));
+ fail("small & diff");
+ } catch (AssertionError e) {}
+
+
}
/**
@@ -529,4 +743,5 @@ public class TestCloudPivotFacet extends
* @see #assertPivotCountsAreCorrect(SolrParams,SolrParams)
*/
private int countNumFoundChecks = 0;
+
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java Thu Nov 6 20:15:52 2014
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
@@ -665,7 +666,80 @@ public class DistributedFacetPivotLargeT
"facet.pivot","place_s,company_t",
FacetParams.FACET_OVERREQUEST_RATIO, "0",
FacetParams.FACET_OVERREQUEST_COUNT, "0");
-
+
+ doTestDeepPivotStats();
+ }
+
+ private void doTestDeepPivotStats() throws Exception {
+
+ QueryResponse rsp = query("q", "*:*",
+ "rows", "0",
+ "facet", "true",
+ "facet.pivot","{!stats=s1}place_s,company_t",
+ "stats", "true",
+ "stats.field", "{!key=avg_price tag=s1}pay_i");
+
+ List<PivotField> pivots = rsp.getFacetPivot().get("place_s,company_t");
+
+ PivotField cardiffPivotField = pivots.get(0);
+ assertEquals("cardiff", cardiffPivotField.getValue());
+ assertEquals(257, cardiffPivotField.getCount());
+
+ FieldStatsInfo cardiffStatsInfo = cardiffPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", cardiffStatsInfo.getName());
+ assertEquals(0.0, cardiffStatsInfo.getMin());
+ assertEquals(8742.0, cardiffStatsInfo.getMax());
+ assertEquals(257, (long) cardiffStatsInfo.getCount());
+ assertEquals(0, (long) cardiffStatsInfo.getMissing());
+ assertEquals(347554.0, cardiffStatsInfo.getSum());
+ assertEquals(8.20968772E8, cardiffStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(1352.35019455253, (double) cardiffStatsInfo.getMean(), 0.1E-7);
+ assertEquals(1170.86048165857, cardiffStatsInfo.getStddev(), 0.1E-7);
+
+ PivotField bbcCardifftPivotField = cardiffPivotField.getPivot().get(0);
+ assertEquals("bbc", bbcCardifftPivotField.getValue());
+ assertEquals(101, bbcCardifftPivotField.getCount());
+
+ FieldStatsInfo bbcCardifftPivotFieldStatsInfo = bbcCardifftPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals(2400.0, bbcCardifftPivotFieldStatsInfo.getMin());
+ assertEquals(8742.0, bbcCardifftPivotFieldStatsInfo.getMax());
+ assertEquals(101, (long) bbcCardifftPivotFieldStatsInfo.getCount());
+ assertEquals(0, (long) bbcCardifftPivotFieldStatsInfo.getMissing());
+ assertEquals(248742.0, bbcCardifftPivotFieldStatsInfo.getSum());
+ assertEquals(6.52422564E8, bbcCardifftPivotFieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(2462.792079208, (double) bbcCardifftPivotFieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(631.0525860312, bbcCardifftPivotFieldStatsInfo.getStddev(), 0.1E-7);
+
+
+ PivotField placeholder0PivotField = pivots.get(2);
+ assertEquals("0placeholder", placeholder0PivotField.getValue());
+ assertEquals(6, placeholder0PivotField.getCount());
+
+ FieldStatsInfo placeholder0PivotFieldStatsInfo = placeholder0PivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", placeholder0PivotFieldStatsInfo.getName());
+ assertEquals(2000.0, placeholder0PivotFieldStatsInfo.getMin());
+ assertEquals(6400.0, placeholder0PivotFieldStatsInfo.getMax());
+ assertEquals(6, (long) placeholder0PivotFieldStatsInfo.getCount());
+ assertEquals(0, (long) placeholder0PivotFieldStatsInfo.getMissing());
+ assertEquals(22700.0, placeholder0PivotFieldStatsInfo.getSum());
+ assertEquals(1.0105E8, placeholder0PivotFieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(3783.333333333, (double) placeholder0PivotFieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(1741.742422595, placeholder0PivotFieldStatsInfo.getStddev(), 0.1E-7);
+
+ PivotField microsoftPlaceholder0PivotField = placeholder0PivotField.getPivot().get(1);
+ assertEquals("microsoft", microsoftPlaceholder0PivotField.getValue());
+ assertEquals(6, microsoftPlaceholder0PivotField.getCount());
+
+ FieldStatsInfo microsoftPlaceholder0PivotFieldStatsInfo = microsoftPlaceholder0PivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", microsoftPlaceholder0PivotFieldStatsInfo.getName());
+ assertEquals(2000.0, microsoftPlaceholder0PivotFieldStatsInfo.getMin());
+ assertEquals(6400.0, microsoftPlaceholder0PivotFieldStatsInfo.getMax());
+ assertEquals(6, (long) microsoftPlaceholder0PivotFieldStatsInfo.getCount());
+ assertEquals(0, (long) microsoftPlaceholder0PivotFieldStatsInfo.getMissing());
+ assertEquals(22700.0, microsoftPlaceholder0PivotFieldStatsInfo.getSum());
+ assertEquals(1.0105E8, microsoftPlaceholder0PivotFieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(3783.333333333, (double) microsoftPlaceholder0PivotFieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(1741.742422595, microsoftPlaceholder0PivotFieldStatsInfo.getStddev(), 0.1E-7);
}
/**
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java Thu Nov 6 20:15:52 2014
@@ -27,9 +27,8 @@ import java.io.IOException;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -68,9 +67,9 @@ public class DistributedFacetPivotLongTa
// the 5 top foo_s terms have 100 docs each on every shard
for (int i = 0; i < 100; i++) {
for (int j = 0; j < 5; j++) {
- shard0.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j));
- shard1.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j));
- shard2.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j));
+ shard0.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j, "stat_i", j * 13 - i));
+ shard1.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j, "stat_i", j * 3 + i));
+ shard2.add(sdoc("id", getDocNum(), "foo_s", "aaa"+j, "stat_i", i * 7 + j));
}
}
@@ -78,14 +77,14 @@ public class DistributedFacetPivotLongTa
// on both shard0 & shard1 ("bbb_")
for (int i = 0; i < 50; i++) {
for (int j = 0; j < 20; j++) {
- shard0.add(sdoc("id", getDocNum(), "foo_s", "bbb"+j));
- shard1.add(sdoc("id", getDocNum(), "foo_s", "bbb"+j));
+ shard0.add(sdoc("id", getDocNum(), "foo_s", "bbb"+j, "stat_i", 0));
+ shard1.add(sdoc("id", getDocNum(), "foo_s", "bbb"+j, "stat_i", 1));
}
// distracting term appears on only on shard2 50 times
shard2.add(sdoc("id", getDocNum(), "foo_s", "junkA"));
}
// put "bbb0" on shard2 exactly once to sanity check refinement
- shard2.add(sdoc("id", getDocNum(), "foo_s", "bbb0"));
+ shard2.add(sdoc("id", getDocNum(), "foo_s", "bbb0", "stat_i", -2));
// long 'tail' foo_s term appears in 45 docs on every shard
// foo_s:tail is the only term with bar_s sub-pivot terms
@@ -95,11 +94,12 @@ public class DistributedFacetPivotLongTa
// but the top 5 terms are ccc(0-4) -- 7 on each shard
// (4 docs each have junk terms)
String sub_term = (i < 35) ? "ccc"+(i % 5) : ((i < 41) ? "tailB" : "junkA");
- shard0.add(sdoc("id", getDocNum(), "foo_s", "tail", "bar_s", sub_term));
- shard1.add(sdoc("id", getDocNum(), "foo_s", "tail", "bar_s", sub_term));
+ shard0.add(sdoc("id", getDocNum(), "foo_s", "tail", "bar_s", sub_term, "stat_i", i));
+ shard1.add(sdoc("id", getDocNum(), "foo_s", "tail", "bar_s", sub_term, "stat_i", i));
// shard2's top 5 sub-pivot terms are junk only it has with 8 docs each
// and 5 docs that use "tailB"
+ // NOTE: none of these get stat_i ! !
sub_term = (i < 40) ? "junkB"+(i % 5) : "tailB";
shard2.add(sdoc("id", getDocNum(), "foo_s", "tail", "bar_s", sub_term));
}
@@ -175,7 +175,9 @@ public class DistributedFacetPivotLongTa
FacetParams.FACET_OVERREQUEST_RATIO, "0",
"facet", "true",
"facet.limit", "6",
- "facet.pivot", "foo_s,bar_s" )
+ "facet.pivot", "{!stats=sxy}foo_s,bar_s",
+ "stats", "true",
+ "stats.field", "{!tag=sxy}stat_i")
).getFacetPivot().get("foo_s,bar_s");
assertEquals(6, pivots.size());
for (int i = 0; i < 5; i++) {
@@ -183,9 +185,23 @@ public class DistributedFacetPivotLongTa
assertTrue(pivot.toString(), pivot.getValue().toString().startsWith("aaa"));
assertEquals(pivot.toString(), 300, pivot.getCount());
}
- // even w/o the long tail, we should have still asked shard2 to refine bbb0
- assertTrue(pivots.get(5).toString(), pivots.get(5).getValue().equals("bbb0"));
- assertEquals(pivots.get(5).toString(), 101, pivots.get(5).getCount());
+ { // even w/o the long tail, we should have still asked shard2 to refine bbb0
+ pivot = pivots.get(5);
+ assertTrue(pivot.toString(), pivot.getValue().equals("bbb0"));
+ assertEquals(pivot.toString(), 101, pivot.getCount());
+ // basic check of refined stats
+ FieldStatsInfo bbb0Stats = pivot.getFieldStatsInfo().get("stat_i");
+ assertEquals("stat_i", bbb0Stats.getName());
+ assertEquals(-2.0, bbb0Stats.getMin());
+ assertEquals(1.0, bbb0Stats.getMax());
+ assertEquals(101, (long) bbb0Stats.getCount());
+ assertEquals(0, (long) bbb0Stats.getMissing());
+ assertEquals(48.0, bbb0Stats.getSum());
+ assertEquals(0.475247524752475, (double) bbb0Stats.getMean(), 0.1E-7);
+ assertEquals(54.0, bbb0Stats.getSumOfSquares(), 0.1E-7);
+ assertEquals(0.55846323792, bbb0Stats.getStddev(), 0.1E-7);
+ }
+
// with default overrequesting, we should find the correct top 6 including
// long tail and top sub-pivots
@@ -284,6 +300,65 @@ public class DistributedFacetPivotLongTa
assertTrue(pivot.toString(), pivot.getValue().toString().startsWith("ccc"));
assertEquals(pivot.toString(), 14, pivot.getCount());
}
+
+ doTestDeepPivotStats();
+ }
+
+ public void doTestDeepPivotStats() throws Exception {
+ // Deep checking of some Facet stats - no refinement involved here
+
+ List<PivotField> pivots =
+ query("q", "*:*",
+ "shards", getShardsString(),
+ "facet", "true",
+ "rows" , "0",
+ "facet.pivot","{!stats=s1}foo_s,bar_s",
+ "stats", "true",
+ "stats.field", "{!key=avg_price tag=s1}stat_i").getFacetPivot().get("foo_s,bar_s");
+ PivotField aaa0PivotField = pivots.get(0);
+ assertEquals("aaa0", aaa0PivotField.getValue());
+ assertEquals(300, aaa0PivotField.getCount());
+
+ FieldStatsInfo aaa0StatsInfo = aaa0PivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", aaa0StatsInfo.getName());
+ assertEquals(-99.0, aaa0StatsInfo.getMin());
+ assertEquals(693.0, aaa0StatsInfo.getMax());
+ assertEquals(300, (long) aaa0StatsInfo.getCount());
+ assertEquals(0, (long) aaa0StatsInfo.getMissing());
+ assertEquals(34650.0, aaa0StatsInfo.getSum());
+ assertEquals(1.674585E7, aaa0StatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(115.5, (double) aaa0StatsInfo.getMean(), 0.1E-7);
+ assertEquals(206.4493184076, aaa0StatsInfo.getStddev(), 0.1E-7);
+
+ PivotField tailPivotField = pivots.get(5);
+ assertEquals("tail", tailPivotField.getValue());
+ assertEquals(135, tailPivotField.getCount());
+
+ FieldStatsInfo tailPivotFieldStatsInfo = tailPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", tailPivotFieldStatsInfo.getName());
+ assertEquals(0.0, tailPivotFieldStatsInfo.getMin());
+ assertEquals(44.0, tailPivotFieldStatsInfo.getMax());
+ assertEquals(90, (long) tailPivotFieldStatsInfo.getCount());
+ assertEquals(45, (long) tailPivotFieldStatsInfo.getMissing());
+ assertEquals(1980.0, tailPivotFieldStatsInfo.getSum());
+ assertEquals(22.0, (double) tailPivotFieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(58740.0, tailPivotFieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(13.0599310011, tailPivotFieldStatsInfo.getStddev(), 0.1E-7);
+
+ PivotField tailBPivotField = tailPivotField.getPivot().get(0);
+ assertEquals("tailB", tailBPivotField.getValue());
+ assertEquals(17, tailBPivotField.getCount());
+
+ FieldStatsInfo tailBPivotFieldStatsInfo = tailBPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals("avg_price", tailBPivotFieldStatsInfo.getName());
+ assertEquals(35.0, tailBPivotFieldStatsInfo.getMin());
+ assertEquals(40.0, tailBPivotFieldStatsInfo.getMax());
+ assertEquals(12, (long) tailBPivotFieldStatsInfo.getCount());
+ assertEquals(5, (long) tailBPivotFieldStatsInfo.getMissing());
+ assertEquals(450.0, tailBPivotFieldStatsInfo.getSum());
+ assertEquals(37.5, (double) tailBPivotFieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(16910.0, tailBPivotFieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(1.78376517, tailBPivotFieldStatsInfo.getStddev(), 0.1E-7);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java Thu Nov 6 20:15:52 2014
@@ -20,9 +20,11 @@ package org.apache.solr.handler.componen
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
import java.util.List;
import org.apache.solr.BaseDistributedSearchTestCase;
+import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.FacetParams;
@@ -46,20 +48,22 @@ public class DistributedFacetPivotSmallT
// NOTE: we use the literal (4 character) string "null" as a company name
// to help ensure there isn't any bugs where the literal string is treated as if it
// were a true NULL value.
- index(id, 19, "place_t", "cardiff dublin", "company_t", "microsoft polecat");
- index(id, 20, "place_t", "dublin", "company_t", "polecat microsoft null");
+ index(id, 19, "place_t", "cardiff dublin", "company_t", "microsoft polecat", "price_ti", "15");
+ index(id, 20, "place_t", "dublin", "company_t", "polecat microsoft null", "price_ti", "19",
+ // this is the only doc to have solo_* fields, therefore only 1 shard has them
+ // TODO: add enum field - blocked by SOLR-6682
+ "solo_i", 42, "solo_s", "lonely", "solo_dt", "1976-03-06T01:23:45Z");
index(id, 21, "place_t", "london la dublin", "company_t",
- "microsoft fujitsu null polecat");
+ "microsoft fujitsu null polecat", "price_ti", "29");
index(id, 22, "place_t", "krakow london cardiff", "company_t",
- "polecat null bbc");
- index(id, 23, "place_t", "london", "company_t", "");
+ "polecat null bbc", "price_ti", "39");
+ index(id, 23, "place_t", "london", "company_t", "", "price_ti", "29");
index(id, 24, "place_t", "la", "company_t", "");
- index(id, 25, "company_t", "microsoft polecat null fujitsu null bbc");
+ index(id, 25, "company_t", "microsoft polecat null fujitsu null bbc", "price_ti", "59");
index(id, 26, "place_t", "krakow", "company_t", "null");
- index(id, 27, "place_t", "krakow cardiff dublin london la", "company_t",
- "null microsoft polecat bbc fujitsu");
- index(id, 28, "place_t", "cork", "company_t",
- "fujitsu rte");
+ index(id, 27, "place_t", "krakow cardiff dublin london la",
+ "company_t", "null microsoft polecat bbc fujitsu");
+ index(id, 28, "place_t", "cork", "company_t", "fujitsu rte");
commit();
handle.clear();
@@ -332,6 +336,76 @@ public class DistributedFacetPivotSmallT
throw new AssertionError(ae.getMessage() + " <== " + p.toString(), ae);
}
}
+
+ doTestDeepPivotStats();
+
+ doTestPivotStatsFromOneShard();
+ }
+
+ private void doTestDeepPivotStats() throws Exception {
+ SolrParams params = params("q", "*:*", "rows", "0",
+ "facet", "true", "stats", "true",
+ "facet.pivot", "{!stats=s1}place_t,company_t",
+ "stats.field", "{!key=avg_price tag=s1}price_ti");
+ QueryResponse rsp = query(params);
+
+ List<PivotField> placePivots = rsp.getFacetPivot().get("place_t,company_t");
+
+ PivotField dublinPivotField = placePivots.get(0);
+ assertEquals("dublin", dublinPivotField.getValue());
+ assertEquals(4, dublinPivotField.getCount());
+
+ PivotField microsoftPivotField = dublinPivotField.getPivot().get(0);
+ assertEquals("microsoft", microsoftPivotField.getValue());
+ assertEquals(4, microsoftPivotField.getCount());
+
+ FieldStatsInfo dublinMicrosoftStatsInfo = microsoftPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals(15.0, dublinMicrosoftStatsInfo.getMin());
+ assertEquals(29.0, dublinMicrosoftStatsInfo.getMax());
+ assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount());
+ assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing());
+ assertEquals(63.0, dublinMicrosoftStatsInfo.getSum());
+ assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(21.0, (double) dublinMicrosoftStatsInfo.getMean(), 0.1E-7);
+ assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7);
+
+
+ PivotField cardiffPivotField = placePivots.get(2);
+ assertEquals("cardiff", cardiffPivotField.getValue());
+ assertEquals(3, cardiffPivotField.getCount());
+
+ PivotField polecatPivotField = cardiffPivotField.getPivot().get(0);
+ assertEquals("polecat", polecatPivotField.getValue());
+ assertEquals(3, polecatPivotField.getCount());
+
+ FieldStatsInfo cardiffPolecatStatsInfo = polecatPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals(15.0, cardiffPolecatStatsInfo.getMin());
+ assertEquals(39.0, cardiffPolecatStatsInfo.getMax());
+ assertEquals(2, (long) cardiffPolecatStatsInfo.getCount());
+ assertEquals(1, (long) cardiffPolecatStatsInfo.getMissing());
+ assertEquals(54.0, cardiffPolecatStatsInfo.getSum());
+ assertEquals(1746.0, cardiffPolecatStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(27.0, (double) cardiffPolecatStatsInfo.getMean(), 0.1E-7);
+ assertEquals(16.97056274847714, cardiffPolecatStatsInfo.getStddev(), 0.1E-7);
+
+
+ PivotField krakowPivotField = placePivots.get(3);
+ assertEquals("krakow", krakowPivotField.getValue());
+ assertEquals(3, krakowPivotField.getCount());
+
+ PivotField fujitsuPivotField = krakowPivotField.getPivot().get(3);
+ assertEquals("fujitsu", fujitsuPivotField.getValue());
+ assertEquals(1, fujitsuPivotField.getCount());
+
+ FieldStatsInfo krakowFujitsuStatsInfo = fujitsuPivotField.getFieldStatsInfo().get("avg_price");
+ assertEquals(null, krakowFujitsuStatsInfo.getMin());
+ assertEquals(null, krakowFujitsuStatsInfo.getMax());
+ assertEquals(0, (long) krakowFujitsuStatsInfo.getCount());
+ assertEquals(1, (long) krakowFujitsuStatsInfo.getMissing());
+ assertEquals(0.0, krakowFujitsuStatsInfo.getSum());
+ assertEquals(0.0, krakowFujitsuStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(Double.NaN, (double) krakowFujitsuStatsInfo.getMean(), 0.1E-7);
+ assertEquals(0.0, krakowFujitsuStatsInfo.getStddev(), 0.1E-7);
}
// Useful to check for errors, orders lists and does toString() equality check
@@ -351,6 +425,46 @@ public class DistributedFacetPivotSmallT
}
assertEquals(expectedPlacePivots.toString(), placePivots.toString());
}
+
+ /**
+ * sanity check the stat values nested under a pivot when at least one shard
+ * has nothing but missing values for the stat
+ */
+ private void doTestPivotStatsFromOneShard() throws Exception {
+ SolrParams params = params("q", "*:*", "rows", "0",
+ "facet", "true", "stats", "true",
+ "facet.pivot", "{!stats=s1}place_t,company_t",
+ "stats.field", "{!tag=s1}solo_i",
+ "stats.field", "{!tag=s1}solo_s",
+ "stats.field", "{!tag=s1}solo_dt");
+
+ QueryResponse rsp = query(params);
+
+ List<PivotField> placePivots = rsp.getFacetPivot().get("place_t,company_t");
+
+ PivotField placePivot = placePivots.get(0);
+ assertEquals("dublin", placePivot.getValue());
+ assertEquals(4, placePivot.getCount());
+
+ PivotField companyPivot = placePivot.getPivot().get(2);
+ assertEquals("null", companyPivot.getValue());
+ assertEquals(3, companyPivot.getCount());
+
+ for (PivotField pf : new PivotField[] { placePivot, companyPivot }) {
+ assertThereCanBeOnlyOne(pf, pf.getFieldStatsInfo().get("solo_s"), "lonely");
+
+ assertThereCanBeOnlyOne(pf, pf.getFieldStatsInfo().get("solo_i"), 42.0D);
+ assertEquals(pf.getField()+":"+pf.getValue()+": int mean",
+ 42.0D, pf.getFieldStatsInfo().get("solo_i").getMean());
+
+ Object expected = new Date(194923425000L); // 1976-03-06T01:23:45Z
+ assertThereCanBeOnlyOne(pf, pf.getFieldStatsInfo().get("solo_dt"), expected);
+ assertEquals(pf.getField()+":"+pf.getValue()+": date mean",
+ expected, pf.getFieldStatsInfo().get("solo_dt").getMean());
+
+ // TODO: add enum field asserts - blocked by SOLR-6682
+ }
+ }
private void testCountSorting(List<PivotField> pivots) {
Integer lastCount = null;
@@ -365,12 +479,27 @@ public class DistributedFacetPivotSmallT
}
}
+ /**
+ * given a PivotField, a FieldStatsInfo, and a value; asserts that:
+ * <ul>
+ * <li>stat count == 1</li>
+ * <li>stat missing == pivot count - 1</li>
+ * <li>stat min == stat max == value</li>
+ * </ul>
+ */
+ private void assertThereCanBeOnlyOne(PivotField pf, FieldStatsInfo stats, Object val) {
+ String msg = pf.getField() + ":" + pf.getValue();
+ assertEquals(msg + " stats count", 1L, (long) stats.getCount());
+ assertEquals(msg + " stats missing", pf.getCount()-1L, (long) stats.getMissing());
+ assertEquals(msg + " stats min", val, stats.getMin());
+ assertEquals(msg + " stats max", val, stats.getMax());
+ }
+
public static class ComparablePivotField extends PivotField {
- public ComparablePivotField(String f, Object v, int count,
- List<PivotField> pivot) {
- super(f,v,count,pivot);
+ public ComparablePivotField(String f, Object v, int count, List<PivotField> pivot) {
+ super(f,v,count,pivot, null);
}
@Override
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java Thu Nov 6 20:15:52 2014
@@ -806,6 +806,13 @@ public class SolrQuery extends Modifiabl
this.add( StatsParams.STATS_FIELD, field );
}
+
+ public void addGetFieldStatistics( String ... field )
+ {
+ this.set( StatsParams.STATS, true );
+ this.add( StatsParams.STATS_FIELD, field );
+ }
+
public void addStatsFieldFacets( String field, String ... facets )
{
if( field == null ) {
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java Thu Nov 6 20:15:52 2014
@@ -180,6 +180,10 @@ public class FieldStatsInfo implements S
return stddev;
}
+ public Double getSumOfSquares() {
+ return sumOfSquares;
+ }
+
public Map<String, List<FieldStatsInfo>> getFacets() {
return facets;
}
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java?rev=1637204&r1=1637203&r2=1637204&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java Thu Nov 6 20:15:52 2014
@@ -20,6 +20,7 @@ package org.apache.solr.client.solrj.res
import java.io.PrintStream;
import java.io.Serializable;
import java.util.List;
+import java.util.Map;
public class PivotField implements Serializable
{
@@ -27,13 +28,23 @@ public class PivotField implements Seria
final Object _value;
final int _count;
final List<PivotField> _pivot;
-
- public PivotField( String f, Object v, int count, List<PivotField> pivot )
+ final Map<String,FieldStatsInfo> _statsInfo;
+
+ /**
+ * @deprecated Use {@link #PivotField(String,Object,int,List,Map)} with a null <code>statsInfo</code>
+ */
+ @Deprecated
+ public PivotField( String f, Object v, int count, List<PivotField> pivot) {
+ this(f, v, count, pivot, null);
+ }
+
+ public PivotField( String f, Object v, int count, List<PivotField> pivot, Map<String,FieldStatsInfo> statsInfo)
{
_field = f;
_value = v;
_count = count;
_pivot = pivot;
+ _statsInfo = statsInfo;
}
public String getField() {
@@ -52,6 +63,10 @@ public class PivotField implements Seria
return _pivot;
}
+ public Map<String,FieldStatsInfo> getFieldStatsInfo() {
+ return _statsInfo;
+ }
+
@Override
public String toString()
{
@@ -63,7 +78,16 @@ public class PivotField implements Seria
for( int i=0; i<indent; i++ ) {
out.print( " " );
}
- out.println( _field + "=" + _value + " ("+_count+")" );
+ out.print( _field + "=" + _value + " ("+_count+")" );
+ if (null != _statsInfo) {
+ out.print( "->stats:[" );
+ for( FieldStatsInfo fieldStatsInfo : _statsInfo.values() ) {
+ out.print(fieldStatsInfo.toString());
+ out.print(",");
+ }
+ out.print("]");
+ }
+ out.println();
if( _pivot != null ) {
for( PivotField p : _pivot ) {
p.write( out, indent+1 );