You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2015/03/10 18:53:38 UTC
svn commit: r1665639 [2/2] - in /lucene/dev/branches/branch_5x: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/
...
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java?rev=1665639&r1=1665638&r2=1665639&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java Tue Mar 10 17:53:36 2015
@@ -89,19 +89,23 @@ public class DistributedFacetPivotSmallA
handle.put("maxScore", SKIPVAL);
doTestDeepPivotStatsOnString();
- doTestTopStatsWithRefinement();
+
+ doTestTopStatsWithRefinement(true);
+ doTestTopStatsWithRefinement(false);
}
/**
* we need to ensure that stats never "overcount" the values from a single shard
* even if we hit that shard with a refinement request
*/
- private void doTestTopStatsWithRefinement() throws Exception {
-
-
+ private void doTestTopStatsWithRefinement(final boolean allStats) throws Exception {
+
+ String stat_param = allStats ?
+ "{!tag=s1}foo_i" : "{!tag=s1 min=true max=true count=true missing=true}foo_i";
+
ModifiableSolrParams coreParams = params("q", "*:*", "rows", "0",
"stats", "true",
- "stats.field", "{!tag=s1}foo_i" );
+ "stats.field", stat_param );
ModifiableSolrParams facetParams = new ModifiableSolrParams(coreParams);
facetParams.add(params("facet", "true",
"facet.limit", "1",
@@ -128,10 +132,18 @@ public class DistributedFacetPivotSmallA
assertEquals(msg, 91.0, fieldStatsInfo.getMax());
assertEquals(msg, 10, (long) fieldStatsInfo.getCount());
assertEquals(msg, 0, (long) fieldStatsInfo.getMissing());
- assertEquals(msg, 248.0, fieldStatsInfo.getSum());
- assertEquals(msg, 15294.0, fieldStatsInfo.getSumOfSquares(), 0.1E-7);
- assertEquals(msg, 24.8, (double) fieldStatsInfo.getMean(), 0.1E-7);
- assertEquals(msg, 31.87405772027709, fieldStatsInfo.getStddev(), 0.1E-7);
+
+ if (allStats) {
+ assertEquals(msg, 248.0, fieldStatsInfo.getSum());
+ assertEquals(msg, 15294.0, fieldStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(msg, 24.8, (double) fieldStatsInfo.getMean(), 0.1E-7);
+ assertEquals(msg, 31.87405772027709, fieldStatsInfo.getStddev(), 0.1E-7);
+ } else {
+ assertNull(msg, fieldStatsInfo.getSum());
+ assertNull(msg, fieldStatsInfo.getSumOfSquares());
+ assertNull(msg, fieldStatsInfo.getMean());
+ assertNull(msg, fieldStatsInfo.getStddev());
+ }
if (params.getBool("facet", false)) {
// if this was a facet request, then the top pivot constraint and pivot
@@ -156,6 +168,12 @@ public class DistributedFacetPivotSmallA
assertEquals(4, (long) dublinMicrosoftStatsInfo.getCount());
assertEquals(0, (long) dublinMicrosoftStatsInfo.getMissing());
+ if (! allStats) {
+ assertNull(msg, dublinMicrosoftStatsInfo.getSum());
+ assertNull(msg, dublinMicrosoftStatsInfo.getSumOfSquares());
+ assertNull(msg, dublinMicrosoftStatsInfo.getMean());
+ assertNull(msg, dublinMicrosoftStatsInfo.getStddev());
+ }
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java?rev=1665639&r1=1665638&r2=1665639&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java Tue Mar 10 17:53:36 2015
@@ -334,16 +334,21 @@ public class DistributedFacetPivotSmallT
}
}
- doTestDeepPivotStats();
+ doTestDeepPivotStats(false); // all price stats
+ doTestDeepPivotStats(true); // just the mean price stat
doTestPivotStatsFromOneShard();
}
- private void doTestDeepPivotStats() throws Exception {
+ /**
+ * @param justMean - only the mean stat is requested/computed
+ */
+ private void doTestDeepPivotStats(boolean justMean) throws Exception {
SolrParams params = params("q", "*:*", "rows", "0",
"facet", "true", "stats", "true",
"facet.pivot", "{!stats=s1}place_t,company_t",
- "stats.field", "{!key=avg_price tag=s1}price_ti");
+ "stats.field", ("{!key=avg_price tag=s1 "+
+ (justMean ? "mean=true" : "") +"}price_ti"));
QueryResponse rsp = query(params);
List<PivotField> placePivots = rsp.getFacetPivot().get("place_t,company_t");
@@ -357,15 +362,24 @@ public class DistributedFacetPivotSmallT
assertEquals(4, microsoftPivotField.getCount());
FieldStatsInfo dublinMicrosoftStatsInfo = microsoftPivotField.getFieldStatsInfo().get("avg_price");
- assertEquals(15.0, dublinMicrosoftStatsInfo.getMin());
- assertEquals(29.0, dublinMicrosoftStatsInfo.getMax());
- assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount());
- assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing());
- assertEquals(63.0, dublinMicrosoftStatsInfo.getSum());
- assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7);
assertEquals(21.0, (double) dublinMicrosoftStatsInfo.getMean(), 0.1E-7);
- assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7);
-
+ if (justMean) {
+ assertNull(dublinMicrosoftStatsInfo.getMin());
+ assertNull(dublinMicrosoftStatsInfo.getMax());
+ assertNull(dublinMicrosoftStatsInfo.getCount());
+ assertNull(dublinMicrosoftStatsInfo.getMissing());
+ assertNull(dublinMicrosoftStatsInfo.getSum());
+ assertNull(dublinMicrosoftStatsInfo.getSumOfSquares());
+ assertNull(dublinMicrosoftStatsInfo.getStddev());
+ } else {
+ assertEquals(15.0, dublinMicrosoftStatsInfo.getMin());
+ assertEquals(29.0, dublinMicrosoftStatsInfo.getMax());
+ assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount());
+ assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing());
+ assertEquals(63.0, dublinMicrosoftStatsInfo.getSum());
+ assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7);
+ }
PivotField cardiffPivotField = placePivots.get(2);
assertEquals("cardiff", cardiffPivotField.getValue());
@@ -376,15 +390,24 @@ public class DistributedFacetPivotSmallT
assertEquals(3, polecatPivotField.getCount());
FieldStatsInfo cardiffPolecatStatsInfo = polecatPivotField.getFieldStatsInfo().get("avg_price");
- assertEquals(15.0, cardiffPolecatStatsInfo.getMin());
- assertEquals(39.0, cardiffPolecatStatsInfo.getMax());
- assertEquals(2, (long) cardiffPolecatStatsInfo.getCount());
- assertEquals(1, (long) cardiffPolecatStatsInfo.getMissing());
- assertEquals(54.0, cardiffPolecatStatsInfo.getSum());
- assertEquals(1746.0, cardiffPolecatStatsInfo.getSumOfSquares(), 0.1E-7);
assertEquals(27.0, (double) cardiffPolecatStatsInfo.getMean(), 0.1E-7);
- assertEquals(16.97056274847714, cardiffPolecatStatsInfo.getStddev(), 0.1E-7);
-
+ if (justMean) {
+ assertNull(cardiffPolecatStatsInfo.getMin());
+ assertNull(cardiffPolecatStatsInfo.getMax());
+ assertNull(cardiffPolecatStatsInfo.getCount());
+ assertNull(cardiffPolecatStatsInfo.getMissing());
+ assertNull(cardiffPolecatStatsInfo.getSum());
+ assertNull(cardiffPolecatStatsInfo.getSumOfSquares());
+ assertNull(cardiffPolecatStatsInfo.getStddev());
+ } else {
+ assertEquals(15.0, cardiffPolecatStatsInfo.getMin());
+ assertEquals(39.0, cardiffPolecatStatsInfo.getMax());
+ assertEquals(2, (long) cardiffPolecatStatsInfo.getCount());
+ assertEquals(1, (long) cardiffPolecatStatsInfo.getMissing());
+ assertEquals(54.0, cardiffPolecatStatsInfo.getSum());
+ assertEquals(1746.0, cardiffPolecatStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(16.97056274847714, cardiffPolecatStatsInfo.getStddev(), 0.1E-7);
+ }
PivotField krakowPivotField = placePivots.get(3);
assertEquals("krakow", krakowPivotField.getValue());
@@ -395,14 +418,25 @@ public class DistributedFacetPivotSmallT
assertEquals(1, fujitsuPivotField.getCount());
FieldStatsInfo krakowFujitsuStatsInfo = fujitsuPivotField.getFieldStatsInfo().get("avg_price");
- assertEquals(null, krakowFujitsuStatsInfo.getMin());
- assertEquals(null, krakowFujitsuStatsInfo.getMax());
- assertEquals(0, (long) krakowFujitsuStatsInfo.getCount());
- assertEquals(1, (long) krakowFujitsuStatsInfo.getMissing());
- assertEquals(0.0, krakowFujitsuStatsInfo.getSum());
- assertEquals(0.0, krakowFujitsuStatsInfo.getSumOfSquares(), 0.1E-7);
assertEquals(Double.NaN, (double) krakowFujitsuStatsInfo.getMean(), 0.1E-7);
- assertEquals(0.0, krakowFujitsuStatsInfo.getStddev(), 0.1E-7);
+ if (justMean) {
+ assertNull(krakowFujitsuStatsInfo.getMin());
+ assertNull(krakowFujitsuStatsInfo.getMax());
+ assertNull(krakowFujitsuStatsInfo.getCount());
+ assertNull(krakowFujitsuStatsInfo.getMissing());
+ assertNull(krakowFujitsuStatsInfo.getSum());
+ assertNull(krakowFujitsuStatsInfo.getSumOfSquares());
+ assertNull(krakowFujitsuStatsInfo.getStddev());
+ } else {
+ assertEquals(null, krakowFujitsuStatsInfo.getMin());
+ assertEquals(null, krakowFujitsuStatsInfo.getMax());
+ assertEquals(0, (long) krakowFujitsuStatsInfo.getCount());
+ assertEquals(1, (long) krakowFujitsuStatsInfo.getMissing());
+ assertEquals(0.0, krakowFujitsuStatsInfo.getSum());
+ assertEquals(0.0, krakowFujitsuStatsInfo.getSumOfSquares(), 0.1E-7);
+ assertEquals(Double.NaN, (double) krakowFujitsuStatsInfo.getMean(), 0.1E-7);
+ assertEquals(0.0, krakowFujitsuStatsInfo.getStddev(), 0.1E-7);
+ }
}
// Useful to check for errors, orders lists and does toString() equality check
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java?rev=1665639&r1=1665638&r2=1665639&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java Tue Mar 10 17:53:36 2015
@@ -158,7 +158,7 @@ public class FacetPivotSmallTest extends
params.add("facet", "true");
params.add("facet.pivot", "{!stats=s1}place_t,company_t");
params.add("stats", "true");
- params.add("stats.field", "{!key=avg_price tag=s1 mean=true}price_ti");
+ params.add("stats.field", "{!key=avg_price tag=s1}price_ti");
SolrQueryRequest req = req(params);
final String statsPrefix = "//lst[@name='facet_counts']/lst[@name='facet_pivot']/arr[@name='place_t,company_t']/lst";
@@ -174,6 +174,8 @@ public class FacetPivotSmallTest extends
dublinMicrosoftStats + "/double[@name='sumOfSquares'][.=1427.0]",
dublinMicrosoftStats + "/double[@name='mean'][.=21.0]",
dublinMicrosoftStats + "/double[@name='stddev'][.=7.211102550927978]",
+ // if new stats are supported, this will break - update test to assert values for each
+ "count(" + dublinMicrosoftStats + "/*)=8",
cardiffPolecatStats + "/double[@name='min'][.=15.0]",
cardiffPolecatStats + "/double[@name='max'][.=39.0]",
@@ -183,6 +185,8 @@ public class FacetPivotSmallTest extends
cardiffPolecatStats + "/double[@name='sumOfSquares'][.=1746.0]",
cardiffPolecatStats + "/double[@name='mean'][.=27.0]",
cardiffPolecatStats + "/double[@name='stddev'][.=16.97056274847714]",
+ // if new stats are supported, this will break - update test to assert values for each
+ "count(" + cardiffPolecatStats + "/*)=8",
krakowFujitsuStats + "/null[@name='min']",
krakowFujitsuStats + "/null[@name='max']",
@@ -191,7 +195,10 @@ public class FacetPivotSmallTest extends
krakowFujitsuStats + "/double[@name='sum'][.=0.0]",
krakowFujitsuStats + "/double[@name='sumOfSquares'][.=0.0]",
krakowFujitsuStats + "/double[@name='mean'][.='NaN']",
- krakowFujitsuStats + "/double[@name='stddev'][.=0.0]"
+ krakowFujitsuStats + "/double[@name='stddev'][.=0.0]",
+ // if new stats are supported, this will break - update test to assert values for each
+ "count(" + krakowFujitsuStats + "/*)=8"
+
);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java?rev=1665639&r1=1665638&r2=1665639&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java Tue Mar 10 17:53:36 2015
@@ -21,6 +21,8 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
+import java.util.Iterator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -29,19 +31,21 @@ import java.util.TimeZone;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
-
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.component.StatsField.Stat;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.AbstractSolrTestCase;
+
+import org.apache.commons.math3.util.Combinations;
+
import org.junit.BeforeClass;
@@ -647,8 +651,20 @@ public class StatsComponentTest extends
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
- assertQ("test string statistics values", req,
- "//null[@name='active_i'][.='']");
+ assertQ("test string statistics values", req
+ ,"//lst[@name='active_i']/long[@name='count'][.='0']"
+ ,"//lst[@name='active_i']/long[@name='missing'][.='4']"
+
+ ,"//lst[@name='active_i']/null[@name='min']"
+ ,"//lst[@name='active_i']/null[@name='max']"
+ ,"//lst[@name='active_i']/double[@name='sum'][.='0.0']"
+ ,"//lst[@name='active_i']/double[@name='sumOfSquares'][.='0.0']"
+ ,"//lst[@name='active_i']/double[@name='stddev'][.='0.0']"
+ ,"//lst[@name='active_i']/double[@name='mean'][.='NaN']"
+ // if new stats are supported, this will break - update test to assert values for each
+ ,"count(//lst[@name='active_i']/*)=8"
+
+ );
}
public void testFieldStatisticsResultsStringFieldAlwaysMissing() throws Exception {
@@ -667,8 +683,15 @@ public class StatsComponentTest extends
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
- assertQ("test string statistics values", req,
- "//null[@name='active_s'][.='']");
+ assertQ("test string statistics values", req
+ ,"//lst[@name='active_s']/long[@name='count'][.='0']"
+ ,"//lst[@name='active_s']/long[@name='missing'][.='4']"
+
+ ,"//lst[@name='active_s']/null[@name='min']"
+ ,"//lst[@name='active_s']/null[@name='max']"
+ // if new stats are supported, this will break - update test to assert values for each
+ ,"count(//lst[@name='active_s']/*)=4"
+ );
}
//SOLR-3160
@@ -688,8 +711,20 @@ public class StatsComponentTest extends
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
- assertQ("test string statistics values", req,
- "//null[@name='active_dt'][.='']");
+ assertQ("test string statistics values", req
+ ,"//lst[@name='active_dt']/long[@name='count'][.='0']"
+ ,"//lst[@name='active_dt']/long[@name='missing'][.='3']"
+
+ ,"//lst[@name='active_dt']/null[@name='min']"
+ ,"//lst[@name='active_dt']/null[@name='max']"
+ ,"//lst[@name='active_dt']/null[@name='mean']"
+ ,"//lst[@name='active_dt']/date[@name='sum'][.='1970-01-01T00:00:00Z']"
+ ,"//lst[@name='active_dt']/double[@name='sumOfSquares'][.='0.0']"
+ ,"//lst[@name='active_dt']/double[@name='stddev'][.='0.0']"
+
+ // if new stats are supported, this will break - update test to assert values for each
+ ,"count(//lst[@name='active_dt']/*)=8"
+ );
}
public void testStatsFacetMultivaluedErrorHandling() throws Exception {
@@ -813,8 +848,8 @@ public class StatsComponentTest extends
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, fieldName);
- args.put("indent", "true");
args.put(StatsParams.STATS_CALC_DISTINCT, "true");
+ args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
@@ -860,8 +895,8 @@ public class StatsComponentTest extends
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, fieldName);
args.put(StatsParams.STATS_FACET, fieldName);
- args.put("indent", "true");
args.put(StatsParams.STATS_CALC_DISTINCT, "true");
+ args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
@@ -987,27 +1022,107 @@ public class StatsComponentTest extends
assertU(commit());
- Map<String, String> args = new HashMap<>();
- args.put(CommonParams.Q, "*:*");
- args.put(StatsParams.STATS, "true");
- args.put(StatsParams.STATS_FIELD, fieldName);
- args.put(StatsParams.STATS_CALC_DISTINCT, "true");
- args.put("indent", "true");
- SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+ final SolrParams baseParams = params(CommonParams.Q, "*:*",
+ "indent", "true",
+ StatsParams.STATS, "true");
+
+ SolrQueryRequest req1 = req(baseParams,
+ StatsParams.STATS_CALC_DISTINCT, "true",
+ StatsParams.STATS_FIELD, fieldName);
+ SolrQueryRequest req2 = req(baseParams,
+ StatsParams.STATS_FIELD,
+ "{!min=true, max=true, count=true, sum=true, mean=true, stddev=true, sumOfSquares=true, missing=true, calcdistinct=true}" + fieldName);
+
+ for (SolrQueryRequest req : new SolrQueryRequest[] { req1, req2 }) {
+ assertQ("test status on docValues and multiValued: " + req.toString(), req
+ , "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']"
+ , "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']"
+ , "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']"
+ , "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']"
+ , "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']"
+ , "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']"
+ , "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']"
+ , "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']"
+ , "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']"
+ // always comes along with countDistinct
+ , "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/float)=9"
+ // if new default stats are added, this will break - update test to assert values for each
+ ,"count(//lst[@name='" + fieldName + "']/*)=10"
+ );
+ }
+ }
- assertQ("test min/max on docValues and multiValued", req
- , "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']"
- , "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']"
- , "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']"
- , "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']"
- , "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']"
- , "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']"
- , "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']"
- , "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']"
- , "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']");
+ public void testEnumFieldTypeStatus() throws Exception {
+ clearIndex();
+
+ String fieldName = "severity";
+ assertU(adoc("id", "0", fieldName, "Not Available"));
+ assertU(adoc("id", "1", fieldName, "Not Available"));
+ assertU(adoc("id", "2", fieldName, "Not Available"));
+ assertU(adoc("id", "3", fieldName, "Not Available"));
+ assertU(adoc("id", "4", fieldName, "Not Available"));
+ assertU(adoc("id", "5", fieldName, "Low"));
+ assertU(adoc("id", "6", fieldName, "Low"));
+ assertU(adoc("id", "7", fieldName, "Low"));
+ assertU(adoc("id", "8", fieldName, "Low"));
+ assertU(adoc("id", "9", fieldName, "Medium"));
+ assertU(adoc("id", "10", fieldName, "Medium"));
+ assertU(adoc("id", "11", fieldName, "Medium"));
+ assertU(adoc("id", "12", fieldName, "High"));
+ assertU(adoc("id", "13", fieldName, "High"));
+ assertU(adoc("id", "14", fieldName, "Critical"));
+
+
+ for (int i = 20; i <= 30; i++) {
+ assertU(adoc("id", "" + i));
+ }
- }
+ assertU(commit());
+
+ assertQ("enum", req("q","*:*", "stats", "true", "stats.field", fieldName)
+ , "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']"
+ , "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']"
+ , "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']"
+ , "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");
+
+
+ assertQ("enum calcdistinct", req("q","*:*", "stats", "true", "stats.field", fieldName,
+ StatsParams.STATS_CALC_DISTINCT, "true")
+ , "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']"
+ , "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']"
+ , "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']"
+ , "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='5']"
+ , "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/*)=5"
+ , "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");
+
+
+ final String pre = "//lst[@name='stats_fields']/lst[@name='"+fieldName+"']/lst[@name='facets']/lst[@name='severity']";
+ assertQ("enum + stats.facet", req("q","*:*", "stats", "true", "stats.field", fieldName,
+ "stats.facet", fieldName)
+ , pre + "/lst[@name='High']/str[@name='min'][.='High']"
+ , pre + "/lst[@name='High']/str[@name='max'][.='High']"
+ , pre + "/lst[@name='High']/long[@name='count'][.='2']"
+ , pre + "/lst[@name='High']/long[@name='missing'][.='0']"
+ , pre + "/lst[@name='Low']/str[@name='min'][.='Low']"
+ , pre + "/lst[@name='Low']/str[@name='max'][.='Low']"
+ , pre + "/lst[@name='Low']/long[@name='count'][.='4']"
+ , pre + "/lst[@name='Low']/long[@name='missing'][.='0']"
+ , pre + "/lst[@name='Medium']/str[@name='min'][.='Medium']"
+ , pre + "/lst[@name='Medium']/str[@name='max'][.='Medium']"
+ , pre + "/lst[@name='Medium']/long[@name='count'][.='3']"
+ , pre + "/lst[@name='Medium']/long[@name='missing'][.='0']"
+ , pre + "/lst[@name='Not Available']/str[@name='min'][.='Not Available']"
+ , pre + "/lst[@name='Not Available']/str[@name='max'][.='Not Available']"
+ , pre + "/lst[@name='Not Available']/long[@name='count'][.='5']"
+ , pre + "/lst[@name='Not Available']/long[@name='missing'][.='0']"
+ , pre + "/lst[@name='Critical']/str[@name='min'][.='Critical']"
+ , pre + "/lst[@name='Critical']/str[@name='max'][.='Critical']"
+ , pre + "/lst[@name='Critical']/long[@name='count'][.='1']"
+ , pre + "/lst[@name='Critical']/long[@name='missing'][.='0']"
+ );
+ }
+
private Doc createDocValuesDocument(List<FldType> types, String fieldName, String id, Comparable... values) throws Exception {
Doc doc = createDoc(types);
doc.getValues("id").set(0, id);
@@ -1020,30 +1135,302 @@ public class StatsComponentTest extends
return cat_docValues;
}
+ public void testIndividualStatLocalParams() throws Exception {
+ final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
+
+ assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
+ assertU(commit());
+
+ // some quick sanity check assertions...
+
+ // trivial check that we only get the exact 2 we ask for
+ assertQ("ask for and get only 2 stats",
+ req("q","*:*", "stats", "true",
+ "stats.field", "{!key=k mean=true min=true}a_i")
+ , kpre + "double[@name='mean'][.='9.0']"
+ , kpre + "double[@name='min'][.='9.0']"
+ , "count(" + kpre + "*)=2"
+ );
+
+ // for stats that are true/false, sanity check false does it's job
+ assertQ("min=true & max=false: only min should come back",
+ req("q","*:*", "stats", "true",
+ "stats.field", "{!key=k max=false min=true}a_i")
+ , kpre + "double[@name='min'][.='9.0']"
+ , "count(" + kpre + "*)=1"
+ );
+ assertQ("min=false: localparam stat means ignore default set, "+
+ "but since only local param is false no stats should be returned",
+ req("q","*:*", "stats", "true",
+ "stats.field", "{!key=k min=false}a_i")
+ , "count(" + kpre + "*)=0"
+ );
+
+ double sum = 0;
+ double sumOfSquares = 0;
+ final int count = 20;
+ for (int i = 0; i < count; i++) {
+ assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i", String.valueOf(i%10), "foo_t", "how now brown cow"));
+ sum+=i%10;
+ sumOfSquares+=(i%10)*(i%10);
+ }
+
+ assertU(commit());
+
+ EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
+
+ Map<Stat, String> expectedStats = new HashMap<>();
+ expectedStats.put(Stat.min, "0.0");
+ expectedStats.put(Stat.max, "9.0");
+ expectedStats.put(Stat.missing, "0");
+ expectedStats.put(Stat.sum, String.valueOf(sum));
+ expectedStats.put(Stat.count, String.valueOf(count));
+ expectedStats.put(Stat.mean, String.valueOf(sum/count));
+ expectedStats.put(Stat.sumOfSquares, String.valueOf(sumOfSquares));
+ expectedStats.put(Stat.stddev, String.valueOf(Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)))));
+ expectedStats.put(Stat.calcdistinct, "10");
+
+ Map<Stat, String> expectedType = new HashMap<>();
+ expectedType.put(Stat.min, "double");
+ expectedType.put(Stat.max, "double");
+ expectedType.put(Stat.missing, "long");
+ expectedType.put(Stat.sum, "double");
+ expectedType.put(Stat.count, "long");
+ expectedType.put(Stat.mean, "double");
+ expectedType.put(Stat.sumOfSquares, "double");
+ expectedType.put(Stat.stddev, "double");
+ expectedType.put(Stat.calcdistinct, "long");
+
+ // canary in the coal mine
+ assertEquals("size of expectedStats doesn't match all known stats; " +
+ "enum was updated w/o updating test?",
+ expectedStats.size(), allStats.size());
+ assertEquals("size of expectedType doesn't match all known stats; " +
+ "enum was updated w/o updating test?",
+ expectedType.size(), allStats.size());
+
+ // whitebox test: explicitly ask for isShard=true with an individual stat
+ for (Stat stat : expectedStats.keySet()) {
+ EnumSet<Stat> distribDeps = stat.getDistribDeps();
+
+ StringBuilder exclude = new StringBuilder();
+ List<String> testParas = new ArrayList<String>(distribDeps.size() + 2);
+ int calcdistinctFudge = 0;
+
+ for (Stat perShardStat : distribDeps ){
+ String key = perShardStat.toString();
+ if (perShardStat.equals(Stat.calcdistinct)) {
+ // this abomination breaks all the rules - uses a diff response key and triggers
+ // the additional "distinctValues" stat
+ key = "countDistinct";
+ calcdistinctFudge++;
+ testParas.add("count(" + kpre + "arr[@name='distinctValues']/*)=10");
+ }
+ testParas.add(kpre + expectedType.get(perShardStat) +
+ "[@name='" + key + "'][.='" + expectedStats.get(perShardStat) + "']");
+ // even if we go out of our way to exclude the dependent stats,
+ // the shard should return them since they are a dependency for the requested stat
+ exclude.append(perShardStat + "=false ");
+ }
+ testParas.add("count(" + kpre + "*)=" + (distribDeps.size() + calcdistinctFudge));
+
+ assertQ("ask for only "+stat+", with isShard=true, and expect only deps: " + distribDeps,
+ req("q", "*:*", "isShard", "true", "stats", "true",
+ "stats.field", "{!key=k " + exclude + stat + "=true}a_i")
+ , testParas.toArray(new String[testParas.size()])
+ );
+ }
+
+ // test all the possible combinations (of all possible sizes) of stats params
+ for (int numParams = 1; numParams <= allStats.size(); numParams++) {
+ for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
+
+ // EnumSets use natural ordering, we want to randomize the order of the params
+ List<Stat> combo = new ArrayList<Stat>(set);
+ Collections.shuffle(combo, random());
+
+ StringBuilder paras = new StringBuilder("{!key=k ");
+ List<String> testParas = new ArrayList<String>(numParams + 2);
+
+ int calcdistinctFudge = 0;
+ for (Stat stat : combo) {
+ String key = stat.toString();
+ if (stat.equals(Stat.calcdistinct)) {
+ // this abomination breaks all the rules - uses a diff response key and triggers
+ // the additional "distinctValues" stat
+ key = "countDistinct";
+ calcdistinctFudge++;
+ testParas.add("count(" + kpre + "arr[@name='distinctValues']/*)=10");
+ }
+ paras.append(stat + "=true ");
+ testParas.add(kpre + expectedType.get(stat) + "[@name='" + key + "'][.='" + expectedStats.get(stat) + "']");
+ }
+
+ paras.append("}a_i");
+ testParas.add("count(" + kpre + "*)=" + (combo.size() + calcdistinctFudge));
+
+ assertQ("ask for an get only: "+ combo,
+ req("q","*:*", "stats", "true",
+ "stats.field", paras.toString())
+ , testParas.toArray(new String[testParas.size()])
+ );
+ }
+ }
+
+ }
-// public void testOtherFacetStatsResult() throws Exception {
-//
-// assertU(adoc("id", "1", "stats_tls_dv", "10", "active_i", "1"));
-// assertU(adoc("id", "2", "stats_tls_dv", "20", "active_i", "1"));
-// assertU(commit());
-// assertU(adoc("id", "3", "stats_tls_dv", "30", "active_i", "2"));
-// assertU(adoc("id", "4", "stats_tls_dv", "40", "active_i", "2"));
-// assertU(commit());
-//
-// final String pre = "//lst[@name='stats_fields']/lst[@name='stats_tls_dv']/lst[@name='facets']/lst[@name='active_i']";
-//
-// assertQ("test value for active_s=true", req("q", "*:*", "stats", "true", "stats.field", "stats_tls_dv", "stats.facet", "active_i","indent", "true")
-// , "*[count("+pre+")=1]"
-// , pre+"/lst[@name='1']/double[@name='min'][.='10.0']"
-// , pre+"/lst[@name='1']/double[@name='max'][.='20.0']"
-// , pre+"/lst[@name='1']/double[@name='sum'][.='30.0']"
-// , pre+"/lst[@name='1']/long[@name='count'][.='2']"
-// , pre+"/lst[@name='1']/long[@name='missing'][.='0']"
-// , pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']"
-// , "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2"
-// , pre+"/lst[@name='1']/double[@name='sumOfSquares'][.='500.0']"
-// , pre+"/lst[@name='1']/double[@name='mean'][.='15.0']"
-// , pre+"/lst[@name='1']/double[@name='stddev'][.='7.0710678118654755']"
-// );
-// }
+ // Test for Solr-6349
+ public void testCalcDistinctStats() throws Exception {
+ final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
+ final String min = "count(" + kpre +"/double[@name='min'])";
+ final String countDistinct = "count(" + kpre +"/long[@name='countDistinct'])";
+ final String distinctValues = "count(" + kpre +"/arr[@name='distinctValues'])";
+
+ final int count = 20;
+ for (int i = 0; i < count; i++) {
+ assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i",
+ String.valueOf(i % 10), "foo_t", "how now brown cow"));
+ }
+
+ assertU(commit());
+
+ String[] baseParams = new String[] { "q", "*:*", "stats", "true","indent", "true" };
+
+ for (SolrParams p : new SolrParams[] {
+ params("stats.field", "{!key=k}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k}a_i"),
+ params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "true",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k}a_i"),
+ params("stats.field", "{!key=k min='true'}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "true",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
+ "stats.field", "{!key=k min='true' calcdistinct='false'}a_i"),
+ }) {
+
+ assertQ("min is either default or explicitly requested; "+
+ "countDistinct & distinctValues either default or explicitly prevented"
+ , req(p, baseParams)
+ , min + "=1"
+ , countDistinct + "=0"
+ , distinctValues + "=0");
+ }
+
+ for (SolrParams p : new SolrParams[] {
+ params("stats.calcdistinct", "true",
+ "stats.field", "{!key=k}a_i"),
+ params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
+ "stats.field", "{!key=k}a_i"),
+ params("stats.calcdistinct", "false",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
+ "stats.field", "{!key=k}a_i"),
+ params("stats.calcdistinct", "false ",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ params("stats.calcdistinct", "false ",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ }) {
+
+ assertQ("min is either default or explicitly requested; " +
+ "countDistinct & distinctValues explicitly requested"
+ , req(p, baseParams)
+ , min + "=1"
+ , countDistinct + "=1"
+ , distinctValues + "=1");
+ }
+
+ for (SolrParams p : new SolrParams[] {
+ params("stats.field", "{!key=k calcdistinct=true}a_i"),
+
+ params("stats.calcdistinct", "true",
+ "stats.field", "{!key=k min='false'}a_i"),
+
+ params("stats.calcdistinct", "true",
+ "stats.field", "{!key=k max='true' min='false'}a_i"),
+
+ params("stats.calcdistinct", "false",
+ "stats.field", "{!key=k calcdistinct=true}a_i"),
+ params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k calcdistinct=true}a_i"),
+ params("stats.calcdistinct", "false",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k calcdistinct=true}a_i"),
+ params("stats.calcdistinct", "false",
+ "f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k min='false' calcdistinct=true}a_i"),
+ }) {
+
+ assertQ("min is explicitly excluded; " +
+ "countDistinct & distinctValues explicitly requested"
+ , req(p, baseParams)
+ , min + "=0"
+ , countDistinct + "=1"
+ , distinctValues + "=1");
+ }
+
+ for (SolrParams p : new SolrParams[] {
+ params(StatsParams.STATS_CALC_DISTINCT, "true",
+ "stats.field", "{!key=k min=true}a_i"),
+ params("f.a_i.stats.calcdistinct", "true",
+ "stats.field", "{!key=k min=true}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "false",
+ "f.a_i.stats.calcdistinct", "true",
+ "stats.field", "{!key=k min=true}a_i"),
+ params("f.a_i.stats.calcdistinct", "false",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "false",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ params(StatsParams.STATS_CALC_DISTINCT, "false",
+ "f.a_i.stats.calcdistinct", "false",
+ "stats.field", "{!key=k min=true calcdistinct=true}a_i"),
+ }) {
+
+ assertQ("min is explicitly requested; " +
+ "countDistinct & distinctValues explicitly requested"
+ , req(p, baseParams)
+ , min + "=1"
+ , countDistinct + "=1"
+ , distinctValues + "=1");
+ }
+ }
+
+ /**
+ * given a comboSize and an EnumSet of Stats, generates iterators that produce every possible
+ * enum combination of that size
+ */
+ public static final class StatSetCombinations implements Iterable<EnumSet<Stat>> {
+ // we need an array so we can do fixed index offset lookups
+ private final Stat[] all;
+ private final Combinations intCombos;
+ public StatSetCombinations(int comboSize, EnumSet<Stat> universe) {
+ // NOTE: should not need to sort, EnumSet uses natural ordering
+ all = universe.toArray(new Stat[universe.size()]);
+ intCombos = new Combinations(all.length, comboSize);
+ }
+ public Iterator<EnumSet<Stat>> iterator() {
+ return new Iterator<EnumSet<Stat>>() {
+ final Iterator<int[]> wrapped = intCombos.iterator();
+ public void remove() {
+ wrapped.remove();
+ }
+ public boolean hasNext() {
+ return wrapped.hasNext();
+ }
+ public EnumSet<Stat> next() {
+ EnumSet<Stat> result = EnumSet.noneOf(Stat.class);
+ int[] indexes = wrapped.next();
+ for (int i = 0; i < indexes.length; i++) {
+ result.add(all[indexes[i]]);
+ }
+ return result;
+ }
+ };
+ }
+ }
+
}
Modified: lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1665639&r1=1665638&r2=1665639&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Tue Mar 10 17:53:36 2015
@@ -750,7 +750,7 @@ public abstract class SolrTestCaseJ4 ext
try {
String m = (null == message) ? "" : message + " ";
String response = h.query(req);
-
+
if (req.getParams().getBool("facet", false)) {
// add a test to ensure that faceting did not throw an exception
// internally, where it would be added to facet_counts/exception
@@ -1888,6 +1888,7 @@ public abstract class SolrTestCaseJ4 ext
FileUtils.copyFile(new File(top, "open-exchange-rates.json"), new File(subHome, "open-exchange-rates.json"));
FileUtils.copyFile(new File(top, "protwords.txt"), new File(subHome, "protwords.txt"));
FileUtils.copyFile(new File(top, "schema.xml"), new File(subHome, "schema.xml"));
+ FileUtils.copyFile(new File(top, "enumsConfig.xml"), new File(subHome, "enumsConfig.xml"));
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
FileUtils.copyFile(new File(top, "solrconfig.xml"), new File(subHome, "solrconfig.xml"));
FileUtils.copyFile(new File(top, "stopwords.txt"), new File(subHome, "stopwords.txt"));