You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dp...@apache.org on 2017/10/17 18:08:34 UTC
[15/20] lucene-solr:branch_7x: SOLR-11145,
SOLR-11146: Added comprehensive unit tests for Analytics Component
2.0 as well as analytics bug fixes.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/SolrAnalyticsTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/SolrAnalyticsTestCase.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/SolrAnalyticsTestCase.java
new file mode 100644
index 0000000..993c1d3
--- /dev/null
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/SolrAnalyticsTestCase.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.analytics;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.solr.JSONTestUtil;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.junit.BeforeClass;
+
+public class SolrAnalyticsTestCase extends SolrCloudTestCase {
+ private static final double DEFAULT_DELTA = .0000001;
+
+ protected static final String COLLECTIONORALIAS = "collection1";
+ protected static final int TIMEOUT = DEFAULT_TIMEOUT;
+ protected static final String id = "id";
+
+ private static UpdateRequest cloudReq;
+
+ @BeforeClass
+ public static void setupCollection() throws Exception {
+ // Single-sharded core
+ initCore("solrconfig-analytics.xml", "schema-analytics.xml");
+ h.update("<delete><query>*:*</query></delete>");
+
+ // Solr Cloud
+ configureCluster(4)
+ .addConfig("conf", configset("cloud-analytics"))
+ .configure();
+
+ CollectionAdminRequest.createCollection(COLLECTIONORALIAS, "conf", 2, 1).process(cluster.getSolrClient());
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTIONORALIAS, cluster.getSolrClient().getZkStateReader(),
+ false, true, TIMEOUT);
+
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ cloudReq = new UpdateRequest();
+ }
+
+ protected static void cleanIndex() throws Exception {
+ h.update("<delete><query>*:*</query></delete>");
+
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ }
+
+ protected static void addDoc(List<String> fieldsAndValues) {
+ assertU(adoc(fieldsAndValues.toArray(new String[0])));
+ cloudReq.add(fieldsAndValues.toArray(new String[0]));
+ }
+
+ protected static void commitDocs() {
+ assertU(commit());
+ try {
+ cloudReq.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ cloudReq = new UpdateRequest();
+ }
+
+ private void testResults(SolrParams params, String analyticsRequest, String... tests) {
+ String coreJson = queryCoreJson(params);
+ Object cloudObj = queryCloudObject(params);
+
+ for (String test : tests) {
+ if (test == null || test.length()==0) continue;
+ // Single-Sharded
+ String err = null;
+ try {
+ err = JSONTestUtil.match(coreJson, test, DEFAULT_DELTA);
+ } catch (Exception e) {
+ err = e.getMessage();
+ } finally {
+ assertNull("query failed JSON validation. test= Single-Sharded Collection" +
+ "\n error=" + err +
+ "\n expected =" + test +
+ "\n response = " + coreJson +
+ "\n analyticsRequest = " + analyticsRequest, err);
+ }
+
+ // Cloud
+ err = null;
+ try {
+ err = JSONTestUtil.matchObj(cloudObj, test, DEFAULT_DELTA);
+ } catch (Exception e) {
+ err = e.getMessage();
+ } finally {
+ assertNull("query failed JSON validation. test= Solr Cloud Collection" +
+ "\n error=" + err +
+ "\n expected =" + test +
+ "\n response = " + Utils.toJSONString(cloudObj) +
+ "\n analyticsRequest = " + analyticsRequest, err);
+ }
+ }
+ }
+
+ private String queryCoreJson(SolrParams params) {
+ try {
+ return JQ(req(params));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Object queryCloudObject(SolrParams params) {
+ QueryResponse resp;
+ try {
+ cluster.waitForAllNodes(10000);
+ QueryRequest qreq = new QueryRequest(params);
+ resp = qreq.process(cluster.getSolrClient(), COLLECTIONORALIAS);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return convertDatesToStrings(resp.getResponse().asShallowMap());
+ }
+
+ protected void testAnalytics(String analyticsRequest, String... tests) throws IOException, InterruptedException, SolrServerException {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("q", "*:*");
+ params.set("indent", "true");
+ params.set("rows", "0");
+ params.set("wt", "json");
+
+ params.set("analytics", analyticsRequest);
+
+ String[] revisedTests = Arrays.stream(tests).map( test -> "analytics_response/" + test).toArray( size -> new String[size]);
+ testResults(params, analyticsRequest, revisedTests);
+ }
+
+ protected void testExpressions(Map<String, ETP> expressions) throws Exception {
+ StringBuilder analyticsRequest = new StringBuilder("{ \"expressions\": {");
+ String expressionsStr = expressions.entrySet()
+ .stream()
+ .map( entry -> '"' + entry.getKey() + "\":\"" + entry.getValue().expression + '"')
+ .reduce((a,b) -> a + ',' + b)
+ .orElseGet(() -> "");
+ analyticsRequest.append(expressionsStr);
+ analyticsRequest.append("}}");
+
+ String results = expressions.entrySet()
+ .stream()
+ .map( entry -> '"' + entry.getKey() + "\":" + entry.getValue().expectedResultStr())
+ .reduce((a,b) -> a + ',' + b)
+ .orElseGet(() -> "");
+
+ testAnalytics(analyticsRequest.toString(), "results=={"+results+", \"_UNORDERED_\":true}");
+ }
+
+ protected void testGrouping(String grouping,
+ Map<String, String> expressions,
+ Map<String, String> facets,
+ Map<String, List<FVP>> results,
+ boolean sortAscending) throws Exception {
+ testGroupingSorted(grouping,
+ expressions,
+ facets,
+ results,
+ ", 'sort': { 'criteria' : [{'type': 'facetvalue', 'direction': '" + (sortAscending ? "ascending" : "descending") + "'}]}",
+ (fvp1, fvp2) -> fvp1.facetValue.compareTo(fvp2.facetValue),
+ sortAscending);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void testGrouping(String grouping,
+ Map<String, String> expressions,
+ Map<String, String> facets,
+ Map<String, List<FVP>> results,
+ String sortExpression,
+ boolean sortAscending) throws Exception {
+ testGroupingSorted(grouping,
+ expressions,
+ facets,
+ results,
+ ", 'sort': { 'criteria' : [{'type': 'expression', 'expression': '" + sortExpression + "', 'direction': '" + (sortAscending ? "ascending" : "descending") + "'}]}",
+ (fvp1, fvp2) -> fvp1.expectedResults.get(sortExpression).compareTo(fvp2.expectedResults.get(sortExpression)),
+ sortAscending);
+ }
+
+ protected void testGrouping(String grouping,
+ Map<String, String> expressions,
+ Map<String, String> facets,
+ Map<String, List<FVP>> results) throws Exception {
+ testGroupingSorted(grouping,
+ expressions,
+ facets,
+ results,
+ "",
+ (fvp1, fvp2) -> fvp1.compareTo(fvp2),
+ true);
+ }
+
+ private void testGroupingSorted(String grouping,
+ Map<String, String> expressions,
+ Map<String, String> facets,
+ Map<String, List<FVP>> results,
+ String sort,
+ Comparator<FVP> comparator,
+ boolean sortAscending) throws Exception {
+ StringBuilder analyticsRequest = new StringBuilder("{ \"groupings\": { \"" + grouping + "\" : { \"expressions\" : {");
+ String expressionsStr = expressions.entrySet()
+ .stream()
+ .map( entry -> '"' + entry.getKey() + "\":\"" + entry.getValue() + '"')
+ .collect(Collectors.joining(" , "));
+ analyticsRequest.append(expressionsStr);
+ analyticsRequest.append("}, \"facets\": {");
+ String facetsStr = facets.entrySet()
+ .stream()
+ .map( entry -> '"' + entry.getKey() + "\":" + entry.getValue().replaceFirst("}\\s*$", sort) + "}")
+ .collect(Collectors.joining(" , "));
+ analyticsRequest.append(facetsStr);
+ analyticsRequest.append("}}}}");
+
+ String groupingResults = results.entrySet()
+ .stream()
+ .map( facet -> {
+ String resultList = facet.getValue()
+ .stream()
+ .sorted(sortAscending ? comparator : comparator.reversed())
+ .map( fvp -> fvp.toJsonResults() )
+ .collect(Collectors.joining(" , "));
+ return '"' + facet.getKey() + "\" : [ " + resultList + " ]";
+ })
+ .collect(Collectors.joining(" , "));
+
+ testAnalytics(analyticsRequest.toString(), "groupings/" + grouping + "=={"+groupingResults+", \"_UNORDERED_\":true}");
+ }
+
+ private static String resultToJson(Object result) {
+ if (result instanceof String) {
+ return '"' + result.toString() + '"';
+ }
+ return result.toString();
+ }
+
+ /*
+ * Expression Test Pair, contains the expression and the expected result
+ */
+ protected static class ETP {
+ final String expression;
+ final Object expectedResult;
+
+ public ETP(String expression, Object expectedResult) {
+ this.expression = expression;
+ this.expectedResult = expectedResult;
+ }
+
+ public String expectedResultStr() {
+ if (expectedResult instanceof String) {
+ return '"' + expectedResult.toString() + '"';
+ }
+ return expectedResult.toString();
+ }
+ }
+
+ /*
+ * FacetValuePair, contains the expression and the expected result
+ */
+ @SuppressWarnings("rawtypes")
+ protected static class FVP implements Comparable<FVP> {
+ final private int order;
+ final public String facetValue;
+ final public Map<String, Comparable> expectedResults;
+
+ public FVP(int order, String facetValue, Map<String, Comparable> expectedResults) {
+ this.order = order;
+ this.facetValue = facetValue;
+ this.expectedResults = expectedResults;
+ }
+
+ public String toJsonResults() {
+ String valueResults = expectedResults.entrySet()
+ .stream()
+ .map( result -> '"' + result.getKey() + "\":" + resultToJson(result.getValue()))
+ .collect(Collectors.joining(" , "));
+ return "{ \"value\" : \"" + facetValue + "\", \"results\": { " + valueResults + ", \"_UNORDERED_\":true } }";
+ }
+
+ @Override
+ public int compareTo(FVP other) {
+ return Integer.compare(order, other.order);
+ }
+ }
+
+ /*
+ * FacetValuePair, contains the expression and the expected result
+ */
+ @SuppressWarnings("rawtypes")
+ protected static class PivotFVP extends FVP {
+ final public String pivot;
+ final public List<FVP> children;
+
+ public PivotFVP(int order, String pivot, String facetValue, Map<String, Comparable> expectedResults) {
+ super(order, facetValue, expectedResults);
+ this.pivot = pivot;
+ children = new ArrayList<>();
+ }
+
+ public void addChild(FVP child) {
+ children.add(child);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected static Object convertDatesToStrings(Object value) {
+ if (value instanceof Date) {
+ return Instant.ofEpochMilli(((Date)value).getTime()).toString();
+ } else if (value instanceof Map) {
+ ((Map<String,Object>)value).replaceAll( (key, obj) -> convertDatesToStrings(obj));
+ } else if (value instanceof List) {
+ ((List<Object>)value).replaceAll( obj -> convertDatesToStrings(obj));
+ }
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
deleted file mode 100644
index eeecbe6..0000000
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.analytics.expression;
-
-import java.time.Instant;
-import java.util.Date;
-
-import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
-import org.apache.solr.util.DateMathParser;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ExpressionTest extends AbstractAnalyticsStatsTest {
- private static final String fileName = "expressions.txt";
-
- private static final int INT = 71;
- private static final int LONG = 36;
- private static final int FLOAT = 93;
- private static final int DOUBLE = 49;
- private static final int DATE = 12;
- private static final int STRING = 28;
- private static final int NUM_LOOPS = 100;
-
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("solrconfig-analytics.xml", "schema-analytics.xml");
- h.update("<delete><query>*:*</query></delete>");
-
- for (int j = 0; j < NUM_LOOPS; ++j) {
- int i = j % INT;
- long l = j % LONG;
- float f = j % FLOAT;
- double d = j % DOUBLE;
- String dt = (1800 + j % DATE) + "-12-31T23:59:59Z";
- String s = "str" + (j % STRING);
- assertU(adoc("id", "1000" + j, "int_id", "" + i, "long_ld", "" + l, "float_fd", "" + f,
- "double_dd", "" + d, "date_dtd", dt, "string_sd", s));
-
- if (usually()) {
- assertU(commit()); // to have several segments
- }
- }
-
- assertU(commit());
-
- setResponse(h.query(request(fileToStringArr(ExpressionTest.class, fileName))));
- }
-
- @Test
- public void addTest() throws Exception {
- double sumResult = (Double) getStatResult("ar", "sum", VAL_TYPE.DOUBLE);
- double uniqueResult = ((Long) getStatResult("ar", "unique", VAL_TYPE.LONG)).doubleValue();
- double result = (Double) getStatResult("ar", "su", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), sumResult + uniqueResult, result, 0.0);
-
- double meanResult = (Double) getStatResult("ar", "mean", VAL_TYPE.DOUBLE);
- double medianResult = (Double) getStatResult("ar", "median", VAL_TYPE.DOUBLE);
- double countResult = ((Long) getStatResult("ar", "count", VAL_TYPE.LONG)).doubleValue();
- result = (Double) getStatResult("ar", "mcm", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), meanResult + countResult + medianResult, result, 0.0);
- }
-
- @Test
- public void multiplyTest() throws Exception {
- double sumResult = (Double) getStatResult("mr", "sum", VAL_TYPE.DOUBLE);
- double uniqueResult = ((Long) getStatResult("mr", "unique", VAL_TYPE.LONG)).doubleValue();
- double result = (Double) getStatResult("mr", "su", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), sumResult * uniqueResult, result, 0.0);
-
- double meanResult = (Double) getStatResult("mr", "mean", VAL_TYPE.DOUBLE);
- double medianResult = (Double) getStatResult("mr", "median", VAL_TYPE.DOUBLE);
- double countResult = ((Long) getStatResult("mr", "count", VAL_TYPE.LONG)).doubleValue();
- result = (Double) getStatResult("mr", "mcm", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), meanResult * countResult * medianResult, result, 0.0);
- }
-
- @Test
- public void divideTest() throws Exception {
- double sumResult = (Double) getStatResult("dr", "sum", VAL_TYPE.DOUBLE);
- double uniqueResult = ((Long) getStatResult("dr", "unique", VAL_TYPE.LONG)).doubleValue();
- double result = (Double) getStatResult("dr", "su", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), sumResult / uniqueResult, result, 0.0);
-
- double meanResult = (Double) getStatResult("dr", "mean", VAL_TYPE.DOUBLE);
- double countResult = ((Long) getStatResult("dr", "count", VAL_TYPE.LONG)).doubleValue();
- result = (Double) getStatResult("dr", "mc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), meanResult / countResult, result, 0.0);
- }
-
- @Test
- public void powerTest() throws Exception {
- double sumResult = (Double) getStatResult("pr", "sum", VAL_TYPE.DOUBLE);
- double uniqueResult = ((Long) getStatResult("pr", "unique", VAL_TYPE.LONG)).doubleValue();
- double result = (Double) getStatResult("pr", "su", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), Math.pow(sumResult, uniqueResult), result, 0.0);
-
- double meanResult = (Double) getStatResult("pr", "mean", VAL_TYPE.DOUBLE);
- double countResult = ((Long) getStatResult("pr", "count", VAL_TYPE.LONG)).doubleValue();
- result = (Double) getStatResult("pr", "mc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), Math.pow(meanResult, countResult), result, 0.0);
- }
-
- @Test
- public void negateTest() throws Exception {
- double sumResult = (Double) getStatResult("nr", "sum", VAL_TYPE.DOUBLE);
- double result = (Double) getStatResult("nr", "s", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), -1 * sumResult, result, 0.0);
-
- long countResult = ((Long) getStatResult("nr", "count", VAL_TYPE.LONG));
- long lresult = (Long) getStatResult("nr", "c", VAL_TYPE.LONG);
- assertEquals(getRawResponse(), -1 * countResult, lresult, 0.0);
- }
-
- @Test
- public void absoluteValueTest() throws Exception {
- double sumResult = (Double) getStatResult("avr", "sum", VAL_TYPE.DOUBLE);
- double result = (Double) getStatResult("avr", "s", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), sumResult, result, 0.0);
-
- long countResult = ((Long) getStatResult("avr", "count", VAL_TYPE.LONG));
- long lresult = (Long) getStatResult("avr", "c", VAL_TYPE.LONG);
- assertEquals(getRawResponse(), countResult, lresult, 0.0);
- }
-
- @Test
- public void constantNumberTest() throws Exception {
- int result = (Integer) getStatResult("cnr", "c8", VAL_TYPE.INTEGER);
- assertEquals(getRawResponse(), 8, result, 0.0);
-
- float dresult = (Float) getStatResult("cnr", "c10", VAL_TYPE.FLOAT);
- assertEquals(getRawResponse(), 10.0f, dresult, 0.0);
- }
-
- @Test
- public void dateMathTest() throws Exception {
- String math = (String) getStatResult("dmr", "cme", VAL_TYPE.STRING);
- DateMathParser dateMathParser = new DateMathParser();
- dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "median", VAL_TYPE.DATE)).toEpochMilli()));
- String dateMath = (String) getStatResult("dmr", "dmme", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math));
-
- math = (String) getStatResult("dmr", "cma", VAL_TYPE.STRING);
- dateMathParser = new DateMathParser();
- dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "max", VAL_TYPE.DATE)).toEpochMilli()));
- dateMath = (String) getStatResult("dmr", "dmma", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math));
- }
-
- @Test
- public void constantDateTest() throws Exception {
- String date = (String) getStatResult("cdr", "cd1", VAL_TYPE.DATE);
- String str = (String) getStatResult("cdr", "cs1", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), date, str);
-
- date = (String) getStatResult("cdr", "cd2", VAL_TYPE.DATE);
- str = (String) getStatResult("cdr", "cs2", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), date, str);
- }
-
- @Test
- public void constantStringTest() throws Exception {
- String str = (String) getStatResult("csr", "cs1", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), str, "this is the first");
-
- str = (String) getStatResult("csr", "cs2", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), str, "this is the second");
-
- str = (String) getStatResult("csr", "cs3", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), str, "this is the third");
- }
-
- @Test
- public void concatenateTest() throws Exception {
- StringBuilder builder = new StringBuilder();
- builder.append((String) getStatResult("cr", "csmin", VAL_TYPE.STRING));
- builder.append((String) getStatResult("cr", "min", VAL_TYPE.STRING));
- String concat = (String) getStatResult("cr", "ccmin", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), concat, builder.toString());
-
- builder.setLength(0);
- builder.append((String) getStatResult("cr", "csmax", VAL_TYPE.STRING));
- builder.append((String) getStatResult("cr", "max", VAL_TYPE.STRING));
- concat = (String) getStatResult("cr", "ccmax", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), concat, builder.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/FunctionTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/FunctionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/FunctionTest.java
deleted file mode 100644
index dac7921..0000000
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/FunctionTest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.analytics.expression;
-
-
-import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
-import org.apache.solr.analytics.facet.AbstractAnalyticsFacetTest;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class FunctionTest extends AbstractAnalyticsStatsTest {
- static String fileName = "functions.txt";
-
- static public final int INT = 71;
- static public final int LONG = 36;
- static public final int FLOAT = 93;
- static public final int DOUBLE = 49;
- static public final int DATE = 12;
- static public final int STRING = 28;
- static public final int NUM_LOOPS = 100;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("solrconfig-analytics.xml","schema-analytics.xml");
- h.update("<delete><query>*:*</query></delete>");
-
- for (int j = 0; j < NUM_LOOPS; ++j) {
- int i = j%INT+1;
- long l = j%LONG+1;
- float f = j%FLOAT+1;
- double d = j%DOUBLE+1;
- double d0 = j%DOUBLE;
- String dt = (1800+j%DATE) + "-06-30T23:59:59Z";
- String s = "str" + (j%STRING);
-
- double add_if = (double)i+f;
- double add_ldf = (double)l+d+f;
- double mult_if = (double)i*f;
- double mult_ldf = (double)l*d*f;
- double div_if = (double)i/f;
- double div_ld = (double)l/d;
- double pow_if = Math.pow(i,f);
- double pow_ld = Math.pow(l,d);
- int neg_i = i*-1;
- long neg_l = l*-1;
- String dm_2y = (1802+j%DATE) + "-06-30T23:59:59Z";
- String dm_2m = (1800+j%DATE) + "-08-30T23:59:59Z";
- String concat_first = "this is the first"+s;
- String concat_second = "this is the second"+s;
-
- assertU(adoc(AbstractAnalyticsFacetTest.filter("id", "1000" + j, "int_id", "" + i, "long_ld", "" + l, "float_fd", "" + f,
- "double_dd", "" + d, "date_dtd", dt, "string_sd", s,
- "add_if_dd", ""+add_if, "add_ldf_dd", ""+add_ldf, "mult_if_dd", ""+mult_if, "mult_ldf_dd", ""+mult_ldf,
- "div_if_dd", ""+div_if, "div_ld_dd", ""+div_ld, "pow_if_dd", ""+pow_if, "pow_ld_dd", ""+pow_ld,
- "neg_id", ""+neg_i, "neg_ld", ""+neg_l, "const_8_dd", "8", "const_10_dd", "10", "dm_2y_dtd", dm_2y, "dm_2m_dtd", dm_2m,
- "const_00_dtd", "1800-06-30T23:59:59Z", "const_04_dtd", "1804-06-30T23:59:59Z", "const_first_sd", "this is the first", "const_second_sd", "this is the second",
- "concat_first_sd", concat_first, "concat_second_sd", concat_second, "miss_dd", ""+d0 )));
-
-
- if (usually()) {
- assertU(commit()); // to have several segments
- }
- }
-
- assertU(commit());
-
- setResponse(h.query(request(fileToStringArr(FunctionTest.class, fileName))));
- }
-
- @Test
- public void addTest() throws Exception {
- double result = (Double)getStatResult("ar", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("ar", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- // TODO checfk why asserted 2times
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("ar", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("ar", "meanc", VAL_TYPE.DOUBLE);
- assertTrue(result==calculated);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void multiplyTest() throws Exception {
- double result = (Double)getStatResult("mr", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("mr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("mr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("mr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void divideTest() throws Exception {
- Double result = (Double)getStatResult("dr", "sum", VAL_TYPE.DOUBLE);
- Double calculated = (Double)getStatResult("dr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("dr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("dr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void powerTest() throws Exception {
- double result = (Double)getStatResult("pr", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("pr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("pr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("pr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void negateTest() throws Exception {
- double result = (Double)getStatResult("nr", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("nr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("nr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("nr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void absoluteValueTest() throws Exception {
- double result = (Double)getStatResult("avr", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("avr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("avr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("avr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void constantNumberTest() throws Exception {
- double result = (Double)getStatResult("cnr", "sum", VAL_TYPE.DOUBLE);
- double calculated = (Double)getStatResult("cnr", "sumc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- assertEquals(getRawResponse(), result, calculated, 0.0);
-
- result = (Double)getStatResult("cnr", "mean", VAL_TYPE.DOUBLE);
- calculated = (Double)getStatResult("cnr", "meanc", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- assertEquals(getRawResponse(), result, calculated, 0.0);
- }
-
- @Test
- public void dateMathTest() throws Exception {
- String result = (String)getStatResult("dmr", "median", VAL_TYPE.DATE);
- String calculated = (String)getStatResult("dmr", "medianc", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), result, calculated);
-
- result = (String)getStatResult("dmr", "max", VAL_TYPE.DATE);
- calculated = (String)getStatResult("dmr", "maxc", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), result, calculated);
- }
-
- @Test
- public void constantDateTest() throws Exception {
- String result = (String)getStatResult("cdr", "median", VAL_TYPE.DATE);
- String calculated = (String)getStatResult("cdr", "medianc", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), result, calculated);
- assertEquals(getRawResponse(), result, calculated);
-
- result = (String)getStatResult("cdr", "max", VAL_TYPE.DATE);
- calculated = (String)getStatResult("cdr", "maxc", VAL_TYPE.DATE);
- assertEquals(getRawResponse(), result, calculated);
- }
-
- @Test
- public void constantStringTest() throws Exception {
- String result = (String)getStatResult("csr", "min", VAL_TYPE.STRING);
- String calculated = (String)getStatResult("csr", "minc", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), result, calculated);
-
- result = (String)getStatResult("csr", "max", VAL_TYPE.STRING);
- calculated = (String)getStatResult("csr", "maxc", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), result, calculated);
- }
-
- @Test
- public void concatenateTest() throws Exception {
- String result = (String)getStatResult("cr", "min", VAL_TYPE.STRING);
- String calculated = (String)getStatResult("cr", "minc", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), result, calculated);
-
- result = (String)getStatResult("cr", "max", VAL_TYPE.STRING);
- calculated = (String)getStatResult("cr", "maxc", VAL_TYPE.STRING);
- assertEquals(getRawResponse(), result, calculated);
- }
-
- @Test
- public void missingTest() throws Exception {
- double min = (Double)getStatResult("ms", "min", VAL_TYPE.DOUBLE);
- double max = (Double)getStatResult("ms", "max", VAL_TYPE.DOUBLE);
- assertEquals(getRawResponse(), 48.0d, max, 0.0);
- assertEquals(getRawResponse(), 1.0d, min, 0.0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetCloudTest.java
deleted file mode 100644
index 99e5103..0000000
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetCloudTest.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.analytics.facet;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Scanner;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.analytics.util.AnalyticsResponseHeadings;
-import org.apache.solr.analytics.util.MedianCalculator;
-import org.apache.solr.analytics.util.OrdinalCalculator;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.AbstractDistribZkTestBase;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.junit.AfterClass;
-
-public class AbstractAnalyticsFacetCloudTest extends SolrCloudTestCase {
- protected static final HashMap<String,Object> defaults = new HashMap<>();
-
- protected static final String COLLECTIONORALIAS = "collection1";
- protected static final int TIMEOUT = DEFAULT_TIMEOUT;
- protected static final String id = "id";
-
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-analytics"))
- .configure();
-
- CollectionAdminRequest.createCollection(COLLECTIONORALIAS, "conf", 2, 1).process(cluster.getSolrClient());
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTIONORALIAS, cluster.getSolrClient().getZkStateReader(),
- false, true, TIMEOUT);
-
- cleanIndex();
- }
-
- public static void cleanIndex() throws Exception {
- new UpdateRequest()
- .deleteByQuery("*:*")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
- }
-
- protected String latestType = "";
-
- @AfterClass
- public static void afterClassAbstractAnalysis() {
- defaults.clear();
- }
-
- protected NamedList<Object> queryCloudAnalytics(String[] testParams) throws SolrServerException, IOException, InterruptedException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("q", "*:*");
- params.set("indent", "true");
- params.set("olap", "true");
- params.set("rows", "0");
- for (int i = 0; i + 1 < testParams.length;) {
- params.add(testParams[i++], testParams[i++]);
- }
- cluster.waitForAllNodes(10000);
- QueryRequest qreq = new QueryRequest(params);
- QueryResponse resp = qreq.process(cluster.getSolrClient(), COLLECTIONORALIAS);
- return resp.getResponse();
- }
-
- @SuppressWarnings("unchecked")
- protected <T> ArrayList<T> getValueList(NamedList<Object> response, String infoName, String facetType, String facetName, String exprName, boolean includeMissing) {
- NamedList<NamedList<Object>> facetList =
- (NamedList<NamedList<Object>>)response.findRecursive(AnalyticsResponseHeadings.COMPLETED_OLD_HEADER,
- infoName,
- facetType,
- facetName);
-
- ArrayList<T> results = new ArrayList<>();
- facetList.forEach( (name, expressions) -> {
- if (!includeMissing && !name.equals("(MISSING)")) {
- T result = (T)expressions.get(exprName);
- if (result != null)
- results.add(result);
- }
- });
- return results;
- }
-
- protected boolean responseContainsFacetValue(NamedList<Object> response, String infoName, String facetType, String facetName, String facetValue) {
- return null != response.findRecursive(AnalyticsResponseHeadings.COMPLETED_OLD_HEADER,
- infoName,
- facetType,
- facetName,
- facetValue);
- }
-
-
- public static void increment(List<Long> list, int idx){
- Long i = list.remove(idx);
- list.add(idx, i+1);
- }
-
- public static String[] filter(String...args){
- List<String> l = new ArrayList<>();
- for( int i=0; i <args.length; i+=2){
- if( args[i+1].equals("0") || args[i+1].equals("0.0") ||
- args[i+1].equals("1800-12-31T23:59:59Z") || args[i+1].equals("str0") ||
- args[i+1].equals("this is the firststr0") ||
- args[i+1].equals("this is the secondstr0") ){
- continue;
- }
- l.add(args[i]);
- l.add(args[i+1]);
- }
- return l.toArray(new String[0]);
- }
-
- protected void setLatestType(String latestType) {
- this.latestType = latestType;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public <T extends Number & Comparable<T>> ArrayList calculateNumberStat(ArrayList<ArrayList<T>> lists, String stat) {
- ArrayList result;
- if (stat.equals("median")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- result.add(MedianCalculator.getMedian(list));
- }
- } else if (stat.equals("mean")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue();
- }
- result.add(d/list.size());
- }
- } else if (stat.equals("sum")) {
- result = new ArrayList<Double>();
- for (Collection<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue();
- }
- result.add(d);
- }
- } else if (stat.equals("sumOfSquares")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue()*element.doubleValue();
- }
- result.add(d);
- }
- } else if (stat.equals("stddev")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double sum = 0;
- double sumSquares = 0;
- for (T element : list) {
- sum += element.doubleValue();
- sumSquares += element.doubleValue()*element.doubleValue();
- }
- String res = Double.toString(Math.sqrt(sumSquares/list.size()-sum*sum/(list.size()*list.size())));
- result.add(Double.parseDouble(res));
- }
- } else {
- throw new IllegalArgumentException();
- }
- return result;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public <T extends Comparable<T>> ArrayList calculateStat(ArrayList<ArrayList<T>> lists, String stat) {
- ArrayList result;
- if (stat.contains("perc_")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- int ord = (int) Math.ceil(Double.parseDouble(stat.substring(5))/100 * list.size()) - 1;
- ArrayList<Integer> percs = new ArrayList<>(1);
- percs.add(ord);
- OrdinalCalculator.putOrdinalsInPosition(list, percs);
- result.add(list.get(ord));
- }
- } else if (stat.equals("count")) {
- result = new ArrayList<Long>();
- for (List<T> list : lists) {
- result.add((long)list.size());
- }
- } else if (stat.equals("missing")) {
- result = new ArrayList<Long>();
- for (ArrayList<T> list : lists) {
- result.add(calculateMissing(list,latestType));
- }
- } else if (stat.equals("unique")) {
- result = new ArrayList<Long>();
- for (List<T> list : lists) {
- HashSet<T> set = new HashSet<>();
- set.addAll(list);
- result.add((long)set.size());
- }
- } else if (stat.equals("max")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- Collections.sort(list);
- result.add(list.get(list.size()-1));
- }
- } else if (stat.equals("min")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- Collections.sort((List<T>)list);
- result.add(list.get(0));
- }
- } else {
- result = null;
- }
- return result;
- }
-
- @SuppressWarnings("unchecked")
- public <T extends Comparable<T>> Long calculateMissing(ArrayList<T> list, String type) {
- T def = (T)defaults.get(type);
- long miss = 0;
- for (T element : list) {
- if (element.compareTo(def)==0) {
- miss++;
- }
- }
- return Long.valueOf(miss);
- }
-
- public static ModifiableSolrParams fileToParams(Class<?> clazz, String fileName) throws FileNotFoundException {
- InputStream in = clazz.getResourceAsStream(fileName);
- if (in == null) throw new FileNotFoundException("Resource not found: " + fileName);
- Scanner file = new Scanner(in, "UTF-8");
- try {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("q", "*:*");
- params.set("indent", "true");
- params.set("olap", "true");
- params.set("rows", "0");
- while (file.hasNextLine()) {
- String line = file.nextLine();
- line = line.trim();
- if( StringUtils.isBlank(line) || line.startsWith("#")){
- continue;
- }
- String[] param = line.split("=");
- params.set(param[0], param[1]);
- }
- return params;
- } finally {
- IOUtils.closeWhileHandlingException(file, in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
deleted file mode 100644
index ff0e7da..0000000
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.analytics.facet;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Scanner;
-
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.analytics.util.AnalyticsResponseHeadings;
-import org.apache.solr.analytics.util.MedianCalculator;
-import org.apache.solr.analytics.util.OrdinalCalculator;
-import org.apache.solr.request.SolrQueryRequest;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import com.google.common.collect.ObjectArrays;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
-@SolrTestCaseJ4.SuppressPointFields(bugUrl="https://issues.apache.org/jira/browse/SOLR-10949")
-public class AbstractAnalyticsFacetTest extends SolrTestCaseJ4 {
- protected static final HashMap<String,Object> defaults = new HashMap<>();
-
- protected String latestType = "";
-
- private static Document doc;
- private static XPathFactory xPathFact;
- private static String rawResponse;
-
- @BeforeClass
- public static void beforeClassAbstractAnalysis() {
- xPathFact = XPathFactory.newInstance();
- }
-
- @AfterClass
- public static void afterClassAbstractAnalysis() {
- xPathFact = null;
- doc = null;
- rawResponse = null;
- defaults.clear();
- }
-
- protected static void setResponse(String response) throws ParserConfigurationException, IOException, SAXException {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- factory.setNamespaceAware(true); // never forget this!
- DocumentBuilder builder = factory.newDocumentBuilder();
- doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8))));
- rawResponse = response;
- }
-
- protected String getRawResponse() {
- return rawResponse;
- }
-
- protected Node getNode(String xPath) throws XPathExpressionException {
- return (Node)xPathFact.newXPath().compile(xPath).evaluate(doc, XPathConstants.NODE);
- }
- private NodeList getNodes(String n1, String n2, String n3, String element, String n4) throws XPathExpressionException {
- // Construct the XPath expression. The form better not change or all these will fail.
- StringBuilder sb = new StringBuilder("/response/lst[@name='"+AnalyticsResponseHeadings.COMPLETED_OLD_HEADER+"']/lst[@name='").append(n1).append("']");
- sb.append("/lst[@name='").append(n2).append("']");
- sb.append("/lst[@name='").append(n3).append("']");
- sb.append("/lst[@name!='(MISSING)']");
- sb.append("//").append(element).append("[@name='").append(n4).append("']");
- return (NodeList)xPathFact.newXPath().compile(sb.toString()).evaluate(doc, XPathConstants.NODESET);
-
- }
- protected ArrayList<String> getStringList(String n1, String n2, String n3, String element, String n4)
- throws XPathExpressionException {
- ArrayList<String> ret = new ArrayList<>();
- NodeList nodes = getNodes(n1, n2, n3, element, n4);
- for (int idx = 0; idx < nodes.getLength(); ++idx) {
- ret.add(nodes.item(idx).getTextContent());
- }
- return ret;
- }
-
- protected ArrayList<Integer> getIntegerList(String n1, String n2, String n3, String element, String n4)
- throws XPathExpressionException {
- ArrayList<Integer> ret = new ArrayList<>();
- NodeList nodes = getNodes(n1, n2, n3, element, n4);
- for (int idx = 0; idx < nodes.getLength(); ++idx) {
- ret.add(Integer.parseInt(nodes.item(idx).getTextContent()));
- }
- return ret;
- }
- protected ArrayList<Long> getLongList(String n1, String n2, String n3, String element, String n4)
- throws XPathExpressionException {
- ArrayList<Long> ret = new ArrayList<>();
- NodeList nodes = getNodes(n1, n2, n3, element, n4);
- for (int idx = 0; idx < nodes.getLength(); ++idx) {
- ret.add(Long.parseLong(nodes.item(idx).getTextContent()));
- }
- return ret;
- }
- protected ArrayList<Float> getFloatList(String n1, String n2, String n3, String element, String n4)
- throws XPathExpressionException {
- ArrayList<Float> ret = new ArrayList<>();
- NodeList nodes = getNodes(n1, n2, n3, element, n4);
- for (int idx = 0; idx < nodes.getLength(); ++idx) {
- ret.add(Float.parseFloat(nodes.item(idx).getTextContent()));
- }
- return ret;
- }
-
- protected ArrayList<Double> getDoubleList(String n1, String n2, String n3, String element, String n4)
- throws XPathExpressionException {
- ArrayList<Double> ret = new ArrayList<>();
- NodeList nodes = getNodes(n1, n2, n3, element, n4);
- for (int idx = 0; idx < nodes.getLength(); ++idx) {
- ret.add(Double.parseDouble(nodes.item(idx).getTextContent()));
- }
- return ret;
- }
-
-
- public static void increment(List<Long> list, int idx){
- Long i = list.remove(idx);
- list.add(idx, i+1);
- }
-
- public static String[] filter(String...args){
- List<String> l = new ArrayList<>();
- for( int i=0; i <args.length; i+=2){
- if( args[i+1].equals("0") || args[i+1].equals("0.0") ||
- args[i+1].equals("1800-12-31T23:59:59Z") || args[i+1].equals("str0") ||
- args[i+1].equals("this is the firststr0") ||
- args[i+1].equals("this is the secondstr0") ){
- continue;
- }
- l.add(args[i]);
- l.add(args[i+1]);
- }
- return l.toArray(new String[0]);
- }
-
- protected void setLatestType(String latestType) {
- this.latestType = latestType;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public <T extends Number & Comparable<T>> ArrayList calculateNumberStat(ArrayList<ArrayList<T>> lists, String stat) {
- ArrayList result;
- if (stat.equals("median")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- result.add(MedianCalculator.getMedian(list));
- }
- } else if (stat.equals("mean")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue();
- }
- result.add(d/list.size());
- }
- } else if (stat.equals("sum")) {
- result = new ArrayList<Double>();
- for (Collection<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue();
- }
- result.add(d);
- }
- } else if (stat.equals("sumOfSquares")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double d = 0;
- for (T element : list) {
- d += element.doubleValue()*element.doubleValue();
- }
- result.add(d);
- }
- } else if (stat.equals("stddev")) {
- result = new ArrayList<Double>();
- for (List<T> list : lists) {
- double sum = 0;
- double sumSquares = 0;
- for (T element : list) {
- sum += element.doubleValue();
- sumSquares += element.doubleValue()*element.doubleValue();
- }
- String res = Double.toString(Math.sqrt(sumSquares/list.size()-sum*sum/(list.size()*list.size())));
- result.add(Double.parseDouble(res));
- }
- } else {
- throw new IllegalArgumentException();
- }
- return result;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public <T extends Comparable<T>> ArrayList calculateStat(ArrayList<ArrayList<T>> lists, String stat) {
- ArrayList result;
- if (stat.contains("perc_")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- int ord = (int) Math.ceil(Double.parseDouble(stat.substring(5))/100 * list.size()) - 1;
- ArrayList<Integer> percs = new ArrayList<>(1);
- percs.add(ord);
- OrdinalCalculator.putOrdinalsInPosition(list, percs);
- result.add(list.get(ord));
- }
- } else if (stat.equals("count")) {
- result = new ArrayList<Long>();
- for (List<T> list : lists) {
- //if( list.size() == 0) continue;
- result.add((long)list.size());
- }
- } else if (stat.equals("missing")) {
- result = new ArrayList<Long>();
- for (ArrayList<T> list : lists) {
- if( list.size() == 0) continue;
- result.add(calculateMissing(list,latestType));
- }
- } else if (stat.equals("unique")) {
- result = new ArrayList<Long>();
- for (List<T> list : lists) {
- HashSet<T> set = new HashSet<>();
- set.addAll(list);
- result.add((long)set.size());
- }
- } else if (stat.equals("max")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- Collections.sort(list);
- result.add(list.get(list.size()-1));
- }
- } else if (stat.equals("min")) {
- result = new ArrayList<T>();
- for (List<T> list : lists) {
- if( list.size() == 0) continue;
- Collections.sort((List<T>)list);
- result.add(list.get(0));
- }
- } else {
- result = null;
- }
- return result;
- }
-
- @SuppressWarnings("unchecked")
- public <T extends Comparable<T>> Long calculateMissing(ArrayList<T> list, String type) {
- T def = (T)defaults.get(type);
- long miss = 0;
- for (T element : list) {
- if (element.compareTo(def)==0) {
- miss++;
- }
- }
- return Long.valueOf(miss);
- }
-
- public static SolrQueryRequest request(String...args){
- return SolrTestCaseJ4.req( ObjectArrays.concat(BASEPARMS, args,String.class) );
- }
-
- public static final String[] BASEPARMS = new String[]{ "q", "*:*", "indent", "true", "olap", "true", "rows", "0" };
-
-
- public static String[] fileToStringArr(Class<?> clazz, String fileName) throws FileNotFoundException {
- InputStream in = clazz.getResourceAsStream("/solr/analytics/" + fileName);
- if (in == null) throw new FileNotFoundException("Resource not found: " + fileName);
- Scanner file = new Scanner(in, "UTF-8");
- try {
- ArrayList<String> strList = new ArrayList<>();
- while (file.hasNextLine()) {
- String line = file.nextLine();
- if (line.length()<2) {
- continue;
- }
- int commentStart = line.indexOf("//");
- if (commentStart >= 0) {
- line = line.substring(0,commentStart);
- }
- String[] param = line.split("=");
- if (param.length != 2) {
- continue;
- }
- strList.add(param[0]);
- strList.add(param[1]);
- }
- return strList.toArray(new String[0]);
- } finally {
- IOUtils.closeWhileHandlingException(file, in);
- }
- }
-
- protected void removeNodes(String xPath, List<Double> string) throws XPathExpressionException {
- NodeList missingNodes = getNodes(xPath);
- List<Double> result = new ArrayList<Double>();
- for (int idx = 0; idx < missingNodes.getLength(); ++idx) {
- result.add(Double.parseDouble(missingNodes.item(idx).getTextContent()));
- }
- string.removeAll(result);
- }
-
- protected NodeList getNodes(String xPath) throws XPathExpressionException {
- StringBuilder sb = new StringBuilder(xPath);
- return (NodeList) xPathFact.newXPath().compile(sb.toString()).evaluate(doc, XPathConstants.NODESET);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6d1681a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
deleted file mode 100644
index ef09014..0000000
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/facet/FacetSortingTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.analytics.facet;
-
-import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
-import org.apache.solr.analytics.expression.ExpressionTest;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class FacetSortingTest extends AbstractAnalyticsStatsTest {
- private static String fileName = "facetSorting.txt";
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("solrconfig-analytics.xml", "schema-analytics.xml");
- h.update("<delete><query>*:*</query></delete>");
-
- // The data set below is so generated that in bucket corresponding fieldFacet B, double_dd column has null values
- // and in bucket C corresponding to fieldFacet C has null values for column long_ld.
- // FieldFaceting occurs on string_sd field
- assertU(adoc("id", "1001", "string_sd", "A", "double_dd", "" + 3, "long_ld", "" + 1));
- assertU(adoc("id", "1002", "string_sd", "A", "double_dd", "" + 25, "long_ld", "" + 2));
- assertU(adoc("id", "1003", "string_sd", "B", "long_ld", "" + 3));
- assertU(adoc("id", "1004", "string_sd", "B", "long_ld", "" + 4));
- assertU(adoc("id", "1005", "string_sd", "C", "double_dd", "" + 17));
-
- assertU(commit());
- String response = h.query(request(fileToStringArr(ExpressionTest.class, fileName)));
- setResponse(response);
- }
-
- @Test
- public void addTest() throws Exception {
- Double minResult = (Double) getStatResult("ar", "min", VAL_TYPE.DOUBLE);
- Long maxResult = (Long) getStatResult("ar", "max", VAL_TYPE.LONG);
- assertEquals(Double.valueOf(minResult), Double.valueOf(3.0));
- assertEquals(Long.valueOf(maxResult),Long.valueOf(4));
- }
-}