You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dp...@apache.org on 2017/06/25 01:43:04 UTC
lucene-solr:master: SOLR-9981: Performance improvements and bug fixes
for the Analytics component
Repository: lucene-solr
Updated Branches:
refs/heads/master 3b07e7241 -> a5dce163e
SOLR-9981: Performance improvements and bug fixes for the Analytics component
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a5dce163
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a5dce163
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a5dce163
Branch: refs/heads/master
Commit: a5dce163eb09dcc0eb7f7eb81d692bf3d19964a3
Parents: 3b07e72
Author: Dennis Gove <dp...@gmail.com>
Authored: Sat Jun 24 20:22:21 2017 -0400
Committer: Dennis Gove <dp...@gmail.com>
Committed: Sat Jun 24 21:39:01 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 6 +++
.../accumulator/FacetingAccumulator.java | 24 ++++++++++--
.../solr/analytics/expression/Expression.java | 15 ++++++--
.../statistics/MinMaxStatsCollector.java | 23 ++++++++---
.../statistics/NumericStatsCollector.java | 8 ++--
.../StatsCollectorSupplierFactory.java | 14 +++++--
.../analytics/requestFiles/facetSorting.txt | 4 ++
.../analytics/requestXMLFiles/facetSorting.xml | 14 +++++++
.../collection1/conf/solrconfig-analytics.xml | 40 ++++++++++++++++++++
.../solr/collection1/conf/solrconfig-basic.xml | 40 --------------------
.../org/apache/solr/analytics/NoFacetTest.java | 2 +-
.../analytics/expression/ExpressionTest.java | 2 +-
.../facet/AbstractAnalyticsFacetTest.java | 15 ++++++++
.../solr/analytics/facet/FacetSortingTest.java | 40 ++++++++++++++++++++
.../analytics/facet/FieldFacetExtrasTest.java | 2 +-
.../solr/analytics/facet/FieldFacetTest.java | 21 +++++-----
.../solr/analytics/facet/QueryFacetTest.java | 2 +-
.../solr/analytics/facet/RangeFacetTest.java | 2 +-
.../util/valuesource/FunctionTest.java | 2 +-
19 files changed, 203 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 77edbe5..64095fa 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -219,6 +219,12 @@ Optimizations
* SOLR-10727: Avoid polluting the filter cache for certain types of faceting (typically ranges) when
the base docset is empty. (David Smiley)
+* SOLR-9981: Performance improvements and bug fixes for the Analytics component. Performance fix that
+ stops the reading of ALL lucene segments over and again for each stats collector. The AtomicReaderContext
+ that refers to the "current " segment is reused. This fix shows an improvement of about 25% in query
+ time for a dataset of ~10M (=9.8M) records. Given the nature of the fix, the improvement should get
+ better as the dataset increases. Fix for the NPE during comparison (Houston Putman)
+
Other Changes
----------------------
* SOLR-10236: Removed FieldType.getNumericType(). Use getNumberType() instead. (Tomás Fernández Löbbe)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
index d8828a6..92add72 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
@@ -28,7 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.HashMap;
import com.google.common.collect.Iterables;
import org.apache.lucene.index.LeafReaderContext;
@@ -98,7 +98,7 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA
List<RangeFacetRequest> rangeFreqs = request.getRangeFacets();
List<QueryFacetRequest> queryFreqs = request.getQueryFacets();
- this.fieldFacetExpressions = new TreeMap<>();
+ this.fieldFacetExpressions = new HashMap<>();
this.rangeFacetExpressions = new LinkedHashMap<>(rangeFreqs.size());
this.queryFacetExpressions = new LinkedHashMap<>(queryFreqs.size());
this.fieldFacetCollectors = new LinkedHashMap<>(fieldFreqs.size());
@@ -120,8 +120,8 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA
final SchemaField ff = fr.getField();
final FieldFacetAccumulator facc = FieldFacetAccumulator.create(searcher, this, ff);
facetAccumulators.add(facc);
- fieldFacetExpressions.put(freq.getName(), new TreeMap<String, Expression[]>() );
- fieldFacetCollectors.put(freq.getName(), new TreeMap<String,StatsCollector[]>());
+ fieldFacetExpressions.put(freq.getName(), new HashMap<String, Expression[]>() );
+ fieldFacetCollectors.put(freq.getName(), new HashMap<String,StatsCollector[]>());
}
/**
* For each range and query facet request add a bucket to the corresponding
@@ -299,6 +299,22 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA
@Override
public int compare(Entry<String,Expression[]> o1, Entry<String,Expression[]> o2) {
+
+ // Handle nulls. Null is treated as an infinitely big number so that in case of ASCENDING sorts,
+ // Nulls will appear last. In case of DESC sorts, Nulls will appear last.
+ boolean firstIsNull = false;
+ if (o1 == null || o1.getValue() == null || o1.getValue()[comparatorExpressionPlace] == null)
+ firstIsNull = true;
+ boolean secondIsNull = false;
+ if (o2 == null || o2.getValue() == null || o2.getValue()[comparatorExpressionPlace] == null)
+ secondIsNull = true;
+ if (firstIsNull && secondIsNull)
+ return 0;
+ else if (firstIsNull)
+ return 1;
+ else if (secondIsNull)
+ return -1;
+
return comp.compare(o1.getValue()[comparatorExpressionPlace], o2.getValue()[comparatorExpressionPlace]);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java
index ba26d9a..c0c3da3 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java
@@ -29,10 +29,19 @@ public abstract class Expression {
public Comparator<Expression> comparator(final FacetSortDirection direction) {
return (a, b) -> {
- if( direction == FacetSortDirection.ASCENDING ){
- return a.getValue().compareTo(b.getValue());
+ boolean aIsNull = a.getValue() == null;
+ boolean bIsNull = b.getValue() == null;
+
+ if (aIsNull && bIsNull) return 0;
+
+ if( direction == FacetSortDirection.ASCENDING ){ // nulls are last for ASC sort
+ return aIsNull ? 1
+ : bIsNull ? -1
+ : a.getValue().compareTo(b.getValue());
} else {
- return b.getValue().compareTo(a.getValue());
+ return aIsNull ? -1
+ : bIsNull ? 1
+ : b.getValue().compareTo(a.getValue());
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
index c21b045..f41a032 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
@@ -37,19 +37,32 @@ public class MinMaxStatsCollector implements StatsCollector{
protected MutableValue value;
protected final Set<String> statsList;
protected final ValueSource source;
- protected FunctionValues function;
protected ValueFiller valueFiller;
+ private CollectorState state;
- public MinMaxStatsCollector(ValueSource source, Set<String> statsList) {
+ public MinMaxStatsCollector(ValueSource source, Set<String> statsList, CollectorState state) {
this.source = source;
this.statsList = statsList;
+ this.state = state;
}
public void setNextReader(LeafReaderContext context) throws IOException {
- function = source.getValues(null, context);
- valueFiller = function.getValueFiller();
+ state.setNextReader(source, context);
+ valueFiller = state.function.getValueFiller();
value = valueFiller.getValue();
}
+
+ public static class CollectorState {
+ FunctionValues function;
+ LeafReaderContext context = null;
+
+ public void setNextReader(ValueSource source, LeafReaderContext context) throws IOException {
+ if (this.context != context) {
+ this.context = context;
+ this.function = source.getValues(null, context);
+ }
+ }
+ }
public void collect(int doc) throws IOException {
valueFiller.fillValue(doc);
@@ -101,7 +114,7 @@ public class MinMaxStatsCollector implements StatsCollector{
@Override
public FunctionValues getFunction() {
- return function;
+ return state.function;
}
public String valueSourceString() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
index 1f22baa..4b1ca22 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/NumericStatsCollector.java
@@ -29,14 +29,16 @@ public class NumericStatsCollector extends MinMaxStatsCollector {
protected double sumOfSquares = 0;
protected double mean = 0;
protected double stddev = 0;
+ protected CollectorState state;
- public NumericStatsCollector(ValueSource source, Set<String> statsList) {
- super(source, statsList);
+ public NumericStatsCollector(ValueSource source, Set<String> statsList, CollectorState state) {
+ super(source, statsList, state);
+ this.state = state;
}
public void collect(int doc) throws IOException {
super.collect(doc);
- double value = function.doubleVal(doc);
+ double value = state.function.doubleVal(doc);
sum += value;
sumOfSquares += (value * value);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
index e22362d..c7c91d4 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
@@ -33,6 +33,7 @@ import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.solr.analytics.expression.ExpressionFactory;
import org.apache.solr.analytics.request.ExpressionRequest;
+import org.apache.solr.analytics.statistics.MinMaxStatsCollector.CollectorState;
import org.apache.solr.analytics.util.AnalyticsParams;
import org.apache.solr.analytics.util.valuesource.AbsoluteValueDoubleFunction;
import org.apache.solr.analytics.util.valuesource.AddDoubleFunction;
@@ -213,25 +214,32 @@ public class StatsCollectorSupplierFactory {
}
}
}
+
+ final CollectorState states[] = new CollectorState[statsArr.length];
+ for (int count = 0; count < statsArr.length; count++) {
+ states[count] = new CollectorState();
+ }
// Making the Supplier
return new Supplier<StatsCollector[]>() {
+ private final CollectorState collectorState[] = states;
+
public StatsCollector[] get() {
StatsCollector[] collectors = new StatsCollector[statsArr.length];
for (int count = 0; count < statsArr.length; count++) {
if(numericBools[count]){
- StatsCollector sc = new NumericStatsCollector(sourceArr[count], statsArr[count]);
+ StatsCollector sc = new NumericStatsCollector(sourceArr[count], statsArr[count], collectorState[count]);
if(uniqueBools[count]) sc = new UniqueStatsCollector(sc);
if(medianBools[count]) sc = new MedianStatsCollector(sc);
if(percsArr[count]!=null) sc = new PercentileStatsCollector(sc,percsArr[count],percsNames[count]);
collectors[count]=sc;
} else if (dateBools[count]) {
- StatsCollector sc = new MinMaxStatsCollector(sourceArr[count], statsArr[count]);
+ StatsCollector sc = new MinMaxStatsCollector(sourceArr[count], statsArr[count], collectorState[count]);
if(uniqueBools[count]) sc = new UniqueStatsCollector(sc);
if(medianBools[count]) sc = new DateMedianStatsCollector(sc);
if(percsArr[count]!=null) sc = new PercentileStatsCollector(sc,percsArr[count],percsNames[count]);
collectors[count]=sc;
} else {
- StatsCollector sc = new MinMaxStatsCollector(sourceArr[count], statsArr[count]);
+ StatsCollector sc = new MinMaxStatsCollector(sourceArr[count], statsArr[count], collectorState[count]);
if(uniqueBools[count]) sc = new UniqueStatsCollector(sc);
if(medianBools[count]) sc = new MedianStatsCollector(sc);
if(percsArr[count]!=null) sc = new PercentileStatsCollector(sc,percsArr[count],percsNames[count]);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test-files/analytics/requestFiles/facetSorting.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test-files/analytics/requestFiles/facetSorting.txt b/solr/contrib/analytics/src/test-files/analytics/requestFiles/facetSorting.txt
new file mode 100644
index 0000000..4663217
--- /dev/null
+++ b/solr/contrib/analytics/src/test-files/analytics/requestFiles/facetSorting.txt
@@ -0,0 +1,4 @@
+o.ar.s.min=min(double_dd)
+o.ar.s.max=max(long_ld)
+o.ar.ff=string_sd
+o.ar.ff.string_sd.sortstatistic=min
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test-files/analytics/requestXMLFiles/facetSorting.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test-files/analytics/requestXMLFiles/facetSorting.xml b/solr/contrib/analytics/src/test-files/analytics/requestXMLFiles/facetSorting.xml
new file mode 100644
index 0000000..54e698a
--- /dev/null
+++ b/solr/contrib/analytics/src/test-files/analytics/requestXMLFiles/facetSorting.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<analyticsRequestEnvelope stats="true" olap="true">
+ <analyticsRequest>
+ <name>MinMax Request</name>
+ <statistic>
+ <expression>min(double(double_dd))</expression>
+ <name>min</name>
+ </statistic>
+ <statistic>
+ <expression>max(long(long_ld))</expression>
+ <name>max</name>
+ </statistic>
+ </analyticsRequest>
+ </analyticsRequestEnvelope>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml b/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml
new file mode 100644
index 0000000..604bb17
--- /dev/null
+++ b/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <requestHandler name="standard" class="solr.StandardRequestHandler">
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>analytics</str>
+ <str>highlight</str>
+ <str>debug</str>
+ <str>expand</str>
+ </arr>
+ </requestHandler>
+
+ <searchComponent name="analytics" class="org.apache.solr.handler.component.AnalyticsComponent" />
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-basic.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-basic.xml b/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-basic.xml
deleted file mode 100644
index 604bb17..0000000
--- a/solr/contrib/analytics/src/test-files/solr/collection1/conf/solrconfig-basic.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <dataDir>${solr.data.dir:}</dataDir>
- <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
- <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <schemaFactory class="ClassicIndexSchemaFactory"/>
-
- <requestHandler name="standard" class="solr.StandardRequestHandler">
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>analytics</str>
- <str>highlight</str>
- <str>debug</str>
- <str>expand</str>
- </arr>
- </requestHandler>
-
- <searchComponent name="analytics" class="org.apache.solr.handler.component.AnalyticsComponent" />
-
-</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/NoFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/NoFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/NoFacetTest.java
index ae935fb..7b67d00 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/NoFacetTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/NoFacetTest.java
@@ -60,7 +60,7 @@ public class NoFacetTest extends AbstractAnalyticsStatsTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
defaults.put("int_id", new Integer(0));
defaults.put("long_ld", new Long(0));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
index 4a3276b..245ea1c 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
@@ -48,7 +48,7 @@ public class ExpressionTest extends AbstractAnalyticsStatsTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml", "schema-analytics.xml");
+ initCore("solrconfig-analytics.xml", "schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
for (int j = 0; j < NUM_LOOPS; ++j) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
index dab68a3..dd331fd 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
@@ -312,4 +312,19 @@ public class AbstractAnalyticsFacetTest extends SolrTestCaseJ4 {
IOUtils.closeWhileHandlingException(file, in);
}
}
+
+ protected void removeNodes(String xPath, List<Double> string) throws XPathExpressionException {
+ NodeList missingNodes = getNodes(xPath);
+ List<Double> result = new ArrayList<Double>();
+ for (int idx = 0; idx < missingNodes.getLength(); ++idx) {
+ result.add(Double.parseDouble(missingNodes.item(idx).getTextContent()));
+ }
+ string.removeAll(result);
+ }
+
+ protected NodeList getNodes(String xPath) throws XPathExpressionException {
+ StringBuilder sb = new StringBuilder(xPath);
+ return (NodeList) xPathFact.newXPath().compile(sb.toString()).evaluate(doc, XPathConstants.NODESET);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
new file mode 100644
index 0000000..0751b63
--- /dev/null
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
@@ -0,0 +1,40 @@
+package org.apache.solr.analytics.facet;
+
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
+import org.apache.solr.analytics.expression.ExpressionTest;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Appending","Asserting"})
+public class FacetSortingTest extends AbstractAnalyticsStatsTest {
+ private static String fileName = "/analytics/requestFiles/facetSorting.txt";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-analytics.xml", "schema-analytics.xml");
+ h.update("<delete><query>*:*</query></delete>");
+
+ // The data set below is so generated that in bucket corresponding fieldFacet B, double_dd column has null values
+ // and in bucket C corresponding to fieldFacet C has null values for column long_ld.
+ // FieldFaceting occurs on string_sd field
+ assertU(adoc("id", "1001", "string_sd", "A", "double_dd", "" + 3, "long_ld", "" + 1));
+ assertU(adoc("id", "1002", "string_sd", "A", "double_dd", "" + 25, "long_ld", "" + 2));
+ assertU(adoc("id", "1003", "string_sd", "B", "long_ld", "" + 3));
+ assertU(adoc("id", "1004", "string_sd", "B", "long_ld", "" + 4));
+ assertU(adoc("id", "1005", "string_sd", "C", "double_dd", "" + 17));
+
+ assertU(commit());
+ String response = h.query(request(fileToStringArr(ExpressionTest.class, fileName)));
+ System.out.println("Response=" + response);
+ setResponse(response);
+ }
+
+ @Test
+ public void addTest() throws Exception {
+ Double minResult = (Double) getStatResult("ar", "min", VAL_TYPE.DOUBLE);
+ Long maxResult = (Long) getStatResult("ar", "max", VAL_TYPE.LONG);
+ assertEquals(Double.valueOf(minResult), Double.valueOf(3.0));
+ assertEquals(Long.valueOf(maxResult),Long.valueOf(4));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
index fc7df4b..afa0c3c 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
@@ -44,7 +44,7 @@ public class FieldFacetExtrasTest extends AbstractAnalyticsFacetTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
//INT
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
index bf67213..2230b02 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.w3c.dom.Node;
public class FieldFacetTest extends AbstractAnalyticsFacetTest{
@@ -87,7 +88,7 @@ public class FieldFacetTest extends AbstractAnalyticsFacetTest{
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
defaults.put("int", new Integer(0));
@@ -1037,31 +1038,33 @@ public class FieldFacetTest extends AbstractAnalyticsFacetTest{
public void missingFacetTest() throws Exception {
//int MultiDate
String xPath = "/response/lst[@name='stats']/lst[@name='missingf']/lst[@name='fieldFacets']/lst[@name='date_dtdm']/lst[@name='(MISSING)']";
- assertNotNull(getRawResponse(), getNode(xPath));
+ Node missingNodeXPath = getNode(xPath);
+ assertNotNull(getRawResponse(), missingNodeXPath);
ArrayList<Double> string = getDoubleList("missingf", "fieldFacets", "date_dtdm", "double", "mean");
- string.remove(0);
+ super.removeNodes(xPath, string);
ArrayList<Double> stringTest = calculateNumberStat(multiDateTestStart, "mean");
assertEquals(getRawResponse(), string,stringTest);
-
+
//Int String
xPath = "/response/lst[@name='stats']/lst[@name='missingf']/lst[@name='fieldFacets']/lst[@name='string_sd']/lst[@name='(MISSING)']";
- assertNotNull(getRawResponse(), getNode(xPath));
+ missingNodeXPath = getNode(xPath);
+ String missingNodeXPathStr = xPath;
+ assertNotNull(getRawResponse(), missingNodeXPath);
xPath = "/response/lst[@name='stats']/lst[@name='missingf']/lst[@name='fieldFacets']/lst[@name='string_sd']/lst[@name='str0']";
assertNull(getRawResponse(), getNode(xPath));
+
List<Double> intString = getDoubleList("missingf", "fieldFacets", "string_sd", "double", "mean");
- intString.remove(0);
+ removeNodes(missingNodeXPathStr, intString);
ArrayList<Double> intStringTest = calculateNumberStat(intStringTestStart, "mean");
assertEquals(getRawResponse(), intString,intStringTest);
-
+
//Int Date
Collection<Double> intDate = getDoubleList("missingf", "fieldFacets", "date_dtd", "double", "mean");
ArrayList<ArrayList<Double>> intDateMissingTestStart = (ArrayList<ArrayList<Double>>) intDateTestStart.clone();
ArrayList<Double> intDateTest = calculateNumberStat(intDateMissingTestStart, "mean");
assertEquals(getRawResponse(),intDate,intDateTest);
-
-
}
private void checkStddevs(ArrayList<Double> list1, ArrayList<Double> list2) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/QueryFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/QueryFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/QueryFacetTest.java
index f62a82b..99ce02d 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/QueryFacetTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/QueryFacetTest.java
@@ -35,7 +35,7 @@ public class QueryFacetTest extends AbstractAnalyticsFacetTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/RangeFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/RangeFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/RangeFacetTest.java
index cda8202..cf5e36b 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/RangeFacetTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/RangeFacetTest.java
@@ -46,7 +46,7 @@ public class RangeFacetTest extends AbstractAnalyticsFacetTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
//INT
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5dce163/solr/contrib/analytics/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
index 5981cc0..f9a9620 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
@@ -35,7 +35,7 @@ public class FunctionTest extends AbstractAnalyticsStatsTest {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml","schema-analytics.xml");
+ initCore("solrconfig-analytics.xml","schema-analytics.xml");
h.update("<delete><query>*:*</query></delete>");
for (int j = 0; j < NUM_LOOPS; ++j) {