You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2018/04/05 18:00:15 UTC
[1/8] lucene-solr:master: SOLR-12175: Fix TestConfigSetsAPI
Repository: lucene-solr
Updated Branches:
refs/heads/master f1d691127 -> 4137f320a
SOLR-12175: Fix TestConfigSetsAPI
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d2845b03
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d2845b03
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d2845b03
Branch: refs/heads/master
Commit: d2845b033e3d2b7c09c013742a60bc5826c5f5f2
Parents: c58516e
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Apr 5 12:39:33 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Apr 5 14:00:07 2018 -0400
----------------------------------------------------------------------
.../src/test-files/solr/configsets/_default/conf/managed-schema | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d2845b03/solr/core/src/test-files/solr/configsets/_default/conf/managed-schema
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/managed-schema b/solr/core/src/test-files/solr/configsets/_default/conf/managed-schema
index 4168a83..6f4e2ef 100644
--- a/solr/core/src/test-files/solr/configsets/_default/conf/managed-schema
+++ b/solr/core/src/test-files/solr/configsets/_default/conf/managed-schema
@@ -139,6 +139,7 @@
<dynamicField name="*_fs" type="pfloats" indexed="true" stored="true"/>
<dynamicField name="*_d" type="pdouble" indexed="true" stored="true"/>
<dynamicField name="*_ds" type="pdoubles" indexed="true" stored="true"/>
+ <dynamicField name="random_*" type="random"/>
<!-- Type used for data-driven schema, to add a string copy for each text field -->
<dynamicField name="*_str" type="strings" stored="false" docValues="true" indexed="false" />
@@ -211,6 +212,8 @@
<fieldType name="pfloats" class="solr.FloatPointField" docValues="true" multiValued="true"/>
<fieldType name="plongs" class="solr.LongPointField" docValues="true" multiValued="true"/>
<fieldType name="pdoubles" class="solr.DoublePointField" docValues="true" multiValued="true"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
[4/8] lucene-solr:master: SOLR-12183: Refactor Streaming Expression
test cases
Posted by jb...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80375acb/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 7e8b6c6..4d88b4e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.math3.distribution.NormalDistribution;
-import org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -35,20 +33,6 @@ import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.ComparatorOrder;
import org.apache.solr.client.solrj.io.comp.FieldComparator;
-import org.apache.solr.client.solrj.io.eval.AddEvaluator;
-import org.apache.solr.client.solrj.io.eval.AndEvaluator;
-import org.apache.solr.client.solrj.io.eval.EqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.GreaterThanEqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.GreaterThanEvaluator;
-import org.apache.solr.client.solrj.io.eval.IfThenElseEvaluator;
-import org.apache.solr.client.solrj.io.eval.LessThanEqualToEvaluator;
-import org.apache.solr.client.solrj.io.eval.LessThanEvaluator;
-import org.apache.solr.client.solrj.io.eval.NotEvaluator;
-import org.apache.solr.client.solrj.io.eval.OrEvaluator;
-import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
-import org.apache.solr.client.solrj.io.ops.ConcatOperation;
-import org.apache.solr.client.solrj.io.ops.GroupOperation;
-import org.apache.solr.client.solrj.io.ops.ReplaceOperation;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -68,12 +52,6 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-/**
- * All base tests will be done with CloudSolrStream. Under the covers CloudSolrStream uses SolrStream so
- * SolrStream will get fully exercised through these tests.
- *
- **/
-
@Slow
@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
public class StreamExpressionTest extends SolrCloudTestCase {
@@ -293,9 +271,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
-
-
-
@Test
public void testCloudSolrStreamWithZkHost() throws Exception {
@@ -422,197 +397,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertOrder(tuples, 0, 2, 1, 3, 4);
}
- @Test
- public void testUniqueStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("unique", UniqueStream.class);
-
- try {
- // Basic test
- expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f\")");
- stream = new UniqueStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 0, 1, 3, 4);
-
- // Basic test desc
- expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc, a_i desc\"), over=\"a_f\")");
- stream = new UniqueStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 4, 3, 1, 2);
-
- // Basic w/multi comp
- expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f, a_i\")");
- stream = new UniqueStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 5);
- assertOrder(tuples, 0, 2, 1, 3, 4);
-
- // full factory w/multi comp
- stream = factory.constructStream("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f, a_i\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 5);
- assertOrder(tuples, 0, 2, 1, 3, 4);
- } finally {
- solrClientCache.close();
- }
- }
-
- @Test
- public void testSortStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- try {
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("sort", SortStream.class);
-
- // Basic test
- stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assert (tuples.size() == 6);
- assertOrder(tuples, 0, 1, 5, 2, 3, 4);
-
- // Basic test desc
- stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i desc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assert (tuples.size() == 6);
- assertOrder(tuples, 4, 3, 2, 1, 5, 0);
-
- // Basic w/multi comp
- stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc, a_f desc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assert (tuples.size() == 6);
- assertOrder(tuples, 0, 5, 1, 2, 3, 4);
- } finally {
- solrClientCache.close();
- }
- }
-
-
- @Test
- public void testNullStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("null", NullStream.class);
-
- try {
- // Basic test
- stream = factory.constructStream("null(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assertTrue(tuples.size() == 1);
- assertTrue(tuples.get(0).getLong("nullCount") == 6);
- } finally {
- solrClientCache.close();
- }
- }
-
-
- @Test
- public void testParallelNullStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("null", NullStream.class)
- .withFunctionName("parallel", ParallelStream.class);
-
- try {
-
- // Basic test
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"nullCount desc\", null(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), by=\"a_i asc\"))");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assertTrue(tuples.size() == 2);
- long nullCount = 0;
- for (Tuple t : tuples) {
- nullCount += t.getLong("nullCount");
- }
-
- assertEquals(nullCount, 6L);
- } finally {
- solrClientCache.close();
- }
- }
@Test
public void testNulls() throws Exception {
@@ -684,174 +468,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
- @Test
- public void testMergeStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("unique", UniqueStream.class)
- .withFunctionName("merge", MergeStream.class);
-
- // Basic test
- expression = StreamExpressionParser.parse("merge("
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"),"
- + "on=\"a_f asc\")");
-
- stream = new MergeStream(expression, factory);
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- try {
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 0, 1, 3, 4);
-
- // Basic test desc
- expression = StreamExpressionParser.parse("merge("
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
- + "on=\"a_f desc\")");
- stream = new MergeStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 4, 3, 1, 0);
-
- // Basic w/multi comp
- expression = StreamExpressionParser.parse("merge("
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "on=\"a_f asc, a_s asc\")");
- stream = new MergeStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 5);
- assertOrder(tuples, 0, 2, 1, 3, 4);
-
- // full factory w/multi comp
- stream = factory.constructStream("merge("
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "on=\"a_f asc, a_s asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 5);
- assertOrder(tuples, 0, 2, 1, 3, 4);
-
- // full factory w/multi streams
- stream = factory.constructStream("merge("
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "search(" + COLLECTIONORALIAS + ", q=\"id:(2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
- + "on=\"a_f asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 0, 2, 1, 4);
- } finally {
- solrClientCache.close();
- }
- }
-
- @Test
- public void testRankStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("unique", UniqueStream.class)
- .withFunctionName("top", RankStream.class);
- try {
- // Basic test
- expression = StreamExpressionParser.parse("top("
- + "n=3,"
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"),"
- + "sort=\"a_f asc, a_i asc\")");
- stream = new RankStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 3);
- assertOrder(tuples, 0, 2, 1);
-
- // Basic test desc
- expression = StreamExpressionParser.parse("top("
- + "n=2,"
- + "unique("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
- + "over=\"a_f\"),"
- + "sort=\"a_f desc\")");
- stream = new RankStream(expression, factory);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 2);
- assertOrder(tuples, 4, 3);
-
- // full factory
- stream = factory.constructStream("top("
- + "n=4,"
- + "unique("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"),"
- + "over=\"a_f\"),"
- + "sort=\"a_f asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 4);
- assertOrder(tuples, 0, 1, 3, 4);
-
- // full factory, switch order
- stream = factory.constructStream("top("
- + "n=4,"
- + "unique("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc, a_i desc\"),"
- + "over=\"a_f\"),"
- + "sort=\"a_f asc\")");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assert (tuples.size() == 4);
- assertOrder(tuples, 2, 1, 3, 4);
- } finally {
- solrClientCache.close();
- }
- }
@Test
public void testRandomStream() throws Exception {
@@ -983,8 +600,16 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
+
+
+
+
+
+
+
+
@Test
- public void testReducerStream() throws Exception {
+ public void testStatsStream() throws Exception {
new UpdateRequest()
.add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
@@ -998,8216 +623,1105 @@ public class StreamExpressionTest extends SolrCloudTestCase {
.add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
.add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- Tuple t0, t1, t2;
- List<Map> maps0, maps1, maps2;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("reduce", ReducerStream.class)
- .withFunctionName("group", GroupOperation.class);
-
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("stats", StatsStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache cache = new SolrClientCache();
try {
- // basic
- expression = StreamExpressionParser.parse("reduce("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc, a_f asc\"),"
- + "by=\"a_s\","
- + "group(sort=\"a_f desc\", n=\"4\"))");
-
+ streamContext.setSolrClientCache(cache);
+ String expr = "stats(" + COLLECTIONORALIAS + ", q=*:*, sum(a_i), sum(a_f), min(a_i), min(a_f), max(a_i), max(a_f), avg(a_i), avg(a_f), count(*))";
+ expression = StreamExpressionParser.parse(expr);
stream = factory.constructStream(expression);
stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 3);
-
- t0 = tuples.get(0);
- maps0 = t0.getMaps("group");
- assertMaps(maps0, 9, 1, 2, 0);
-
- t1 = tuples.get(1);
- maps1 = t1.getMaps("group");
- assertMaps(maps1, 8, 7, 5, 3);
-
- t2 = tuples.get(2);
- maps2 = t2.getMaps("group");
- assertMaps(maps2, 6, 4);
-
- // basic w/spaces
- expression = StreamExpressionParser.parse("reduce("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc, a_f asc\"),"
- + "by=\"a_s\"," +
- "group(sort=\"a_i asc\", n=\"2\"))");
- stream = factory.constructStream(expression);
- stream.setStreamContext(streamContext);
tuples = getTuples(stream);
- assert (tuples.size() == 3);
-
- t0 = tuples.get(0);
- maps0 = t0.getMaps("group");
- assert (maps0.size() == 2);
-
- assertMaps(maps0, 0, 1);
-
- t1 = tuples.get(1);
- maps1 = t1.getMaps("group");
- assertMaps(maps1, 3, 5);
-
- t2 = tuples.get(2);
- maps2 = t2.getMaps("group");
- assertMaps(maps2, 4, 6);
- } finally {
- solrClientCache.close();
- }
- }
-
-
- @Test
- public void testHavingStream() throws Exception {
-
- SolrClientCache solrClientCache = new SolrClientCache();
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
- .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
- .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
- .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
- .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
- .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ assert (tuples.size() == 1);
- TupleStream stream;
- List<Tuple> tuples;
+ //Test Long and Double Sums
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("having", HavingStream.class)
- .withFunctionName("rollup", RollupStream.class)
- .withFunctionName("sum", SumMetric.class)
- .withFunctionName("and", AndEvaluator.class)
- .withFunctionName("or", OrEvaluator.class)
- .withFunctionName("not", NotEvaluator.class)
- .withFunctionName("gt", GreaterThanEvaluator.class)
- .withFunctionName("lt", LessThanEvaluator.class)
- .withFunctionName("eq", EqualToEvaluator.class)
- .withFunctionName("lteq", LessThanEqualToEvaluator.class)
- .withFunctionName("gteq", GreaterThanEqualToEvaluator.class);
-
- stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), eq(a_i, 9))");
- StreamContext context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ Tuple tuple = tuples.get(0);
- assert(tuples.size() == 1);
- Tuple t = tuples.get(0);
- assertTrue(t.getString("id").equals("9"));
+ Double sumi = tuple.getDouble("sum(a_i)");
+ Double sumf = tuple.getDouble("sum(a_f)");
+ Double mini = tuple.getDouble("min(a_i)");
+ Double minf = tuple.getDouble("min(a_f)");
+ Double maxi = tuple.getDouble("max(a_i)");
+ Double maxf = tuple.getDouble("max(a_f)");
+ Double avgi = tuple.getDouble("avg(a_i)");
+ Double avgf = tuple.getDouble("avg(a_f)");
+ Double count = tuple.getDouble("count(*)");
- stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(eq(a_i, 9),lt(a_i, 10)))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ assertTrue(sumi.longValue() == 70);
+ assertTrue(sumf.doubleValue() == 55.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 7.0D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 10);
- assert(tuples.size() == 1);
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("9"));
- stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), or(eq(a_i, 9),eq(a_i, 8)))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ //Test with shards parameter
+ List<String> shardUrls = TupleStream.getShards(cluster.getZkServer().getZkAddress(), COLLECTIONORALIAS, streamContext);
+ expr = "stats(myCollection, q=*:*, sum(a_i), sum(a_f), min(a_i), min(a_f), max(a_i), max(a_f), avg(a_i), avg(a_f), count(*))";
+ Map<String, List<String>> shardsMap = new HashMap();
+ shardsMap.put("myCollection", shardUrls);
+ StreamContext context = new StreamContext();
+ context.put("shards", shardsMap);
+ context.setSolrClientCache(cache);
+ stream = factory.constructStream(expr);
+ stream.setStreamContext(context);
- assert(tuples.size() == 2);
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("8"));
+ tuples = getTuples(stream);
- t = tuples.get(1);
- assertTrue(t.getString("id").equals("9"));
+ assert (tuples.size() == 1);
+ //Test Long and Double Sums
- stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(eq(a_i, 9),not(eq(a_i, 9))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ tuple = tuples.get(0);
- assert(tuples.size() == 0);
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+ assertTrue(sumi.longValue() == 70);
+ assertTrue(sumf.doubleValue() == 55.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 7.0D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 10);
- stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(lteq(a_i, 9), gteq(a_i, 8)))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ //Execersise the /stream hander
- assert(tuples.size() == 2);
+ //Add the shards http parameter for the myCollection
+ StringBuilder buf = new StringBuilder();
+ for (String shardUrl : shardUrls) {
+ if (buf.length() > 0) {
+ buf.append(",");
+ }
+ buf.append(shardUrl);
+ }
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("8"));
+ ModifiableSolrParams solrParams = new ModifiableSolrParams();
+ solrParams.add("qt", "/stream");
+ solrParams.add("expr", expr);
+ solrParams.add("myCollection.shards", buf.toString());
+ SolrStream solrStream = new SolrStream(shardUrls.get(0), solrParams);
+ tuples = getTuples(solrStream);
+ assert (tuples.size() == 1);
- t = tuples.get(1);
- assertTrue(t.getString("id").equals("9"));
+ tuple =tuples.get(0);
- stream = factory.constructStream("having(rollup(over=a_f, sum(a_i), search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\")), and(eq(sum(a_i), 9),eq(sum(a_i), 9)))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- assert(tuples.size() == 1);
- t = tuples.get(0);
- assertTrue(t.getDouble("a_f") == 10.0D);
+ assertTrue(sumi.longValue() == 70);
+ assertTrue(sumf.doubleValue() == 55.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 7.0D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 10);
+ //Add a negative test to prove that it cannot find slices if shards parameter is removed
- solrClientCache.close();
+ try {
+ ModifiableSolrParams solrParamsBad = new ModifiableSolrParams();
+ solrParamsBad.add("qt", "/stream");
+ solrParamsBad.add("expr", expr);
+ solrStream = new SolrStream(shardUrls.get(0), solrParamsBad);
+ tuples = getTuples(solrStream);
+ throw new Exception("Exception should have been thrown above");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("Collection not found: myCollection"));
+ }
+ } finally {
+ cache.close();
+ }
}
-
@Test
- public void testParallelHavingStream() throws Exception {
-
- SolrClientCache solrClientCache = new SolrClientCache();
+ public void testFacetStream() throws Exception {
new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
- .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
- .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
- .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
- .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
- .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
+
+ String clause;
TupleStream stream;
List<Tuple> tuples;
-
+
StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("having", HavingStream.class)
- .withFunctionName("rollup", RollupStream.class)
- .withFunctionName("sum", SumMetric.class)
- .withFunctionName("and", AndEvaluator.class)
- .withFunctionName("or", OrEvaluator.class)
- .withFunctionName("not", NotEvaluator.class)
- .withFunctionName("gt", GreaterThanEvaluator.class)
- .withFunctionName("lt", LessThanEvaluator.class)
- .withFunctionName("eq", EqualToEvaluator.class)
- .withFunctionName("lteq", LessThanEqualToEvaluator.class)
- .withFunctionName("gteq", GreaterThanEqualToEvaluator.class)
- .withFunctionName("val", RawValueEvaluator.class)
- .withFunctionName("parallel", ParallelStream.class);
-
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), eq(a_i, 9)))");
- StreamContext context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
-
- assert(tuples.size() == 1);
- Tuple t = tuples.get(0);
- assertTrue(t.getString("id").equals("9"));
-
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(eq(a_i, 9),lt(a_i, 10))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
+ .withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress())
+ .withFunctionName("facet", FacetStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class);
+
+ // Basic test
+ clause = "facet("
+ + "collection1, "
+ + "q=\"*:*\", "
+ + "fl=\"a_s,a_i,a_f\", "
+ + "sort=\"a_s asc\", "
+ + "buckets=\"a_s\", "
+ + "bucketSorts=\"sum(a_i) asc\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), sum(a_f), "
+ + "min(a_i), min(a_f), "
+ + "max(a_i), max(a_f), "
+ + "avg(a_i), avg(a_f), "
+ + "count(*)"
+ + ")";
+
+ stream = factory.constructStream(clause);
tuples = getTuples(stream);
- assert(tuples.size() == 1);
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("9"));
+ assert(tuples.size() == 3);
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\",having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), or(eq(a_i, 9),eq(a_i, 8))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ //Test Long and Double Sums
- assert(tuples.size() == 2);
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("8"));
+ Tuple tuple = tuples.get(0);
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+ Double sumf = tuple.getDouble("sum(a_f)");
+ Double mini = tuple.getDouble("min(a_i)");
+ Double minf = tuple.getDouble("min(a_f)");
+ Double maxi = tuple.getDouble("max(a_i)");
+ Double maxf = tuple.getDouble("max(a_f)");
+ Double avgi = tuple.getDouble("avg(a_i)");
+ Double avgf = tuple.getDouble("avg(a_f)");
+ Double count = tuple.getDouble("count(*)");
- t = tuples.get(1);
- assertTrue(t.getString("id").equals("9"));
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(eq(a_i, 9),not(eq(a_i, 9)))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
-
- assert(tuples.size() == 0);
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\",having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(lteq(a_i, 9), gteq(a_i, 8))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
- assert(tuples.size() == 2);
- t = tuples.get(0);
- assertTrue(t.getString("id").equals("8"));
+ //Reverse the Sort.
- t = tuples.get(1);
- assertTrue(t.getString("id").equals("9"));
+ clause = "facet("
+ + "collection1, "
+ + "q=\"*:*\", "
+ + "fl=\"a_s,a_i,a_f\", "
+ + "sort=\"a_s asc\", "
+ + "buckets=\"a_s\", "
+ + "bucketSorts=\"sum(a_i) desc\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), sum(a_f), "
+ + "min(a_i), min(a_f), "
+ + "max(a_i), max(a_f), "
+ + "avg(a_i), avg(a_f), "
+ + "count(*)"
+ + ")";
- stream = factory.constructStream("parallel("+COLLECTIONORALIAS+", workers=2, sort=\"a_f asc\", having(rollup(over=a_f, sum(a_i), search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=a_f)), and(eq(sum(a_i), 9),eq(sum(a_i),9))))");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
+ stream = factory.constructStream(clause);
tuples = getTuples(stream);
- assert(tuples.size() == 1);
- t = tuples.get(0);
- assertTrue(t.getDouble("a_f") == 10.0D);
-
- solrClientCache.close();
- }
+ //Test Long and Double Sums
- @Test
- public void testFetchStream() throws Exception {
+ tuple = tuples.get(0);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- SolrClientCache solrClientCache = new SolrClientCache();//TODO share in @Before ; close in @After ?
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
- .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
- .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
- .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
- .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
- .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- TupleStream stream;
- List<Tuple> tuples;
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("fetch", FetchStream.class);
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- stream = factory.constructStream("fetch("+ COLLECTIONORALIAS +", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), on=\"id=a_i\", batchSize=\"2\", fl=\"subject\")");
- StreamContext context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
- assert(tuples.size() == 10);
- Tuple t = tuples.get(0);
- assertTrue("blah blah blah 0".equals(t.getString("subject")));
- t = tuples.get(1);
- assertTrue("blah blah blah 2".equals(t.getString("subject")));
- t = tuples.get(2);
- assertTrue("blah blah blah 3".equals(t.getString("subject")));
- t = tuples.get(3);
- assertTrue("blah blah blah 4".equals(t.getString("subject")));
- t = tuples.get(4);
- assertTrue("blah blah blah 1".equals(t.getString("subject")));
- t = tuples.get(5);
- assertTrue("blah blah blah 5".equals(t.getString("subject")));
- t = tuples.get(6);
- assertTrue("blah blah blah 6".equals(t.getString("subject")));
- t = tuples.get(7);
- assertTrue("blah blah blah 7".equals(t.getString("subject")));
- t = tuples.get(8);
- assertTrue("blah blah blah 8".equals(t.getString("subject")));
- t = tuples.get(9);
- assertTrue("blah blah blah 9".equals(t.getString("subject")));
-
- //Change the batch size
- stream = factory.constructStream("fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), on=\"id=a_i\", batchSize=\"3\", fl=\"subject\")");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
- tuples = getTuples(stream);
- assert(tuples.size() == 10);
- t = tuples.get(0);
- assertTrue("blah blah blah 0".equals(t.getString("subject")));
- t = tuples.get(1);
- assertTrue("blah blah blah 2".equals(t.getString("subject")));
- t = tuples.get(2);
- assertTrue("blah blah blah 3".equals(t.getString("subject")));
- t = tuples.get(3);
- assertTrue("blah blah blah 4".equals(t.getString("subject")));
- t = tuples.get(4);
- assertTrue("blah blah blah 1".equals(t.getString("subject")));
- t = tuples.get(5);
- assertTrue("blah blah blah 5".equals(t.getString("subject")));
- t = tuples.get(6);
- assertTrue("blah blah blah 6".equals(t.getString("subject")));
- t = tuples.get(7);
- assertTrue("blah blah blah 7".equals(t.getString("subject")));
- t = tuples.get(8);
- assertTrue("blah blah blah 8".equals(t.getString("subject")));
- t = tuples.get(9);
- assertTrue("blah blah blah 9".equals(t.getString("subject")));
-
- // SOLR-10404 test that "hello 99" as a value gets escaped
- new UpdateRequest()
- .add(id, "99", "a1_s", "hello 99", "a2_s", "hello 99", "subject", "blah blah blah 99")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ //Test index sort
+ clause = "facet("
+ + "collection1, "
+ + "q=\"*:*\", "
+ + "fl=\"a_s,a_i,a_f\", "
+ + "sort=\"a_s asc\", "
+ + "buckets=\"a_s\", "
+ + "bucketSorts=\"a_s desc\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), sum(a_f), "
+ + "min(a_i), min(a_f), "
+ + "max(a_i), max(a_f), "
+ + "avg(a_i), avg(a_f), "
+ + "count(*)"
+ + ")";
- stream = factory.constructStream("fetch("+ COLLECTIONORALIAS +", search(" + COLLECTIONORALIAS + ", q=" + id + ":99, fl=\"id,a1_s\", sort=\"id asc\"), on=\"a1_s=a2_s\", fl=\"subject\")");
- context = new StreamContext();
- context.setSolrClientCache(solrClientCache);
- stream.setStreamContext(context);
+ stream = factory.constructStream(clause);
tuples = getTuples(stream);
- assertEquals(1, tuples.size());
- t = tuples.get(0);
- assertTrue("blah blah blah 99".equals(t.getString("subject")));
-
- solrClientCache.close();
- }
+ assert(tuples.size() == 3);
- @Test
- public void testParallelFetchStream() throws Exception {
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
- .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
- .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
- .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
- .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
- .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ tuple = tuples.get(0);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- TupleStream stream;
- List<Tuple> tuples;
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("parallel", ParallelStream.class)
- .withFunctionName("fetch", FetchStream.class);
- try {
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"2\", fl=\"subject\"))");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
- assert (tuples.size() == 10);
- Tuple t = tuples.get(0);
- assertTrue("blah blah blah 0".equals(t.getString("subject")));
- t = tuples.get(1);
- assertTrue("blah blah blah 2".equals(t.getString("subject")));
- t = tuples.get(2);
- assertTrue("blah blah blah 3".equals(t.getString("subject")));
- t = tuples.get(3);
- assertTrue("blah blah blah 4".equals(t.getString("subject")));
- t = tuples.get(4);
- assertTrue("blah blah blah 1".equals(t.getString("subject")));
- t = tuples.get(5);
- assertTrue("blah blah blah 5".equals(t.getString("subject")));
- t = tuples.get(6);
- assertTrue("blah blah blah 6".equals(t.getString("subject")));
- t = tuples.get(7);
- assertTrue("blah blah blah 7".equals(t.getString("subject")));
- t = tuples.get(8);
- assertTrue("blah blah blah 8".equals(t.getString("subject")));
- t = tuples.get(9);
- assertTrue("blah blah blah 9".equals(t.getString("subject")));
-
-
- stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"3\", fl=\"subject\"))");
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
- assert (tuples.size() == 10);
- t = tuples.get(0);
- assertTrue("blah blah blah 0".equals(t.getString("subject")));
- t = tuples.get(1);
- assertTrue("blah blah blah 2".equals(t.getString("subject")));
- t = tuples.get(2);
- assertTrue("blah blah blah 3".equals(t.getString("subject")));
- t = tuples.get(3);
- assertTrue("blah blah blah 4".equals(t.getString("subject")));
- t = tuples.get(4);
- assertTrue("blah blah blah 1".equals(t.getString("subject")));
- t = tuples.get(5);
- assertTrue("blah blah blah 5".equals(t.getString("subject")));
- t = tuples.get(6);
- assertTrue("blah blah blah 6".equals(t.getString("subject")));
- t = tuples.get(7);
- assertTrue("blah blah blah 7".equals(t.getString("subject")));
- t = tuples.get(8);
- assertTrue("blah blah blah 8".equals(t.getString("subject")));
- t = tuples.get(9);
- assertTrue("blah blah blah 9".equals(t.getString("subject")));
- } finally {
- solrClientCache.close();
- }
- }
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
+ //Test index sort
+ clause = "facet("
+ + "collection1, "
+ + "q=\"*:*\", "
+ + "fl=\"a_s,a_i,a_f\", "
+ + "sort=\"a_s asc\", "
+ + "buckets=\"a_s\", "
+ + "bucketSorts=\"a_s asc\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), sum(a_f), "
+ + "min(a_i), min(a_f), "
+ + "max(a_i), max(a_f), "
+ + "avg(a_i), avg(a_f), "
+ + "count(*)"
+ + ")";
+ stream = factory.constructStream(clause);
+ tuples = getTuples(stream);
- @Test
- public void testDaemonStream() throws Exception {
+ assert(tuples.size() == 3);
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
- .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
- .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
- .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
- .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
- .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("rollup", RollupStream.class)
- .withFunctionName("sum", SumMetric.class)
- .withFunctionName("min", MinMetric.class)
- .withFunctionName("max", MaxMetric.class)
- .withFunctionName("avg", MeanMetric.class)
- .withFunctionName("count", CountMetric.class)
- .withFunctionName("daemon", DaemonStream.class);
-
- StreamExpression expression;
- DaemonStream daemonStream;
-
- expression = StreamExpressionParser.parse("daemon(rollup("
- + "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"a_i,a_s\", sort=\"a_s asc\"),"
- + "over=\"a_s\","
- + "sum(a_i)"
- + "), id=\"test\", runInterval=\"1000\", queueSize=\"9\")");
- daemonStream = (DaemonStream)factory.constructStream(expression);
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- daemonStream.setStreamContext(streamContext);
- try {
- //Test Long and Double Sums
-
- daemonStream.open(); // This will start the daemon thread
-
- for (int i = 0; i < 4; i++) {
- Tuple tuple = daemonStream.read(); // Reads from the queue
- String bucket = tuple.getString("a_s");
- Double sumi = tuple.getDouble("sum(a_i)");
-
- //System.out.println("#################################### Bucket 1:"+bucket);
- assertTrue(bucket.equals("hello0"));
- assertTrue(sumi.doubleValue() == 17.0D);
-
- tuple = daemonStream.read();
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
-
- //System.out.println("#################################### Bucket 2:"+bucket);
- assertTrue(bucket.equals("hello3"));
- assertTrue(sumi.doubleValue() == 38.0D);
-
- tuple = daemonStream.read();
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
- //System.out.println("#################################### Bucket 3:"+bucket);
- assertTrue(bucket.equals("hello4"));
- assertTrue(sumi.longValue() == 15);
- }
-
- //Now lets wait until the internal queue fills up
-
- while (daemonStream.remainingCapacity() > 0) {
- try {
- Thread.sleep(1000);
- } catch (Exception e) {
+ tuple = tuples.get(0);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- }
- }
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
- //OK capacity is full, let's index a new doc
- new UpdateRequest()
- .add(id, "10", "a_s", "hello0", "a_i", "1", "a_f", "10")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- //Now lets clear the existing docs in the queue 9, plus 3 more to get passed the run that was blocked. The next run should
- //have the tuples with the updated count.
- for (int i = 0; i < 12; i++) {
- daemonStream.read();
- }
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
- //And rerun the loop. It should have a new count for hello0
- for (int i = 0; i < 4; i++) {
- Tuple tuple = daemonStream.read(); // Reads from the queue
- String bucket = tuple.getString("a_s");
- Double sumi = tuple.getDouble("sum(a_i)");
-
- //System.out.println("#################################### Bucket 1:"+bucket);
- assertTrue(bucket.equals("hello0"));
- assertTrue(sumi.doubleValue() == 18.0D);
-
- tuple = daemonStream.read();
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
-
- //System.out.println("#################################### Bucket 2:"+bucket);
- assertTrue(bucket.equals("hello3"));
- assertTrue(sumi.doubleValue() == 38.0D);
-
- tuple = daemonStream.read();
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
- //System.out.println("#################################### Bucket 3:"+bucket);
- assertTrue(bucket.equals("hello4"));
- assertTrue(sumi.longValue() == 15);
- }
- } finally {
- daemonStream.close(); //This should stop the daemon thread
- solrClientCache.close();
- }
- }
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
- @Test
- public void testTerminatingDaemonStream() throws Exception {
- Assume.assumeTrue(!useAlias);
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
- new UpdateRequest()
- .add(id, "0", "a_s", "hello", "a_i", "0", "a_f", "1")
- .add(id, "2", "a_s", "hello", "a_i", "2", "a_f", "2")
- .add(id, "3", "a_s", "hello", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello", "a_i", "1", "a_f", "5")
- .add(id, "5", "a_s", "hello", "a_i", "10", "a_f", "6")
- .add(id, "6", "a_s", "hello", "a_i", "11", "a_f", "7")
- .add(id, "7", "a_s", "hello", "a_i", "12", "a_f", "8")
- .add(id, "8", "a_s", "hello", "a_i", "13", "a_f", "9")
- .add(id, "9", "a_s", "hello", "a_i", "14", "a_f", "10")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ //Test zero result facets
+ clause = "facet("
+ + "collection1, "
+ + "q=\"blahhh\", "
+ + "fl=\"a_s,a_i,a_f\", "
+ + "sort=\"a_s asc\", "
+ + "buckets=\"a_s\", "
+ + "bucketSorts=\"a_s asc\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), sum(a_f), "
+ + "min(a_i), min(a_f), "
+ + "max(a_i), max(a_f), "
+ + "avg(a_i), avg(a_f), "
+ + "count(*)"
+ + ")";
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("topic", TopicStream.class)
- .withFunctionName("daemon", DaemonStream.class);
+ stream = factory.constructStream(clause);
+ tuples = getTuples(stream);
- StreamExpression expression;
- DaemonStream daemonStream;
+ assert(tuples.size() == 0);
- SolrClientCache cache = new SolrClientCache();
- StreamContext context = new StreamContext();
- context.setSolrClientCache(cache);
- expression = StreamExpressionParser.parse("daemon(topic("+ COLLECTIONORALIAS +","+ COLLECTIONORALIAS +", q=\"a_s:hello\", initialCheckpoint=0, id=\"topic1\", rows=2, fl=\"id\""
- + "), id=test, runInterval=1000, terminate=true, queueSize=50)");
- daemonStream = (DaemonStream)factory.constructStream(expression);
- daemonStream.setStreamContext(context);
-
- List<Tuple> tuples = getTuples(daemonStream);
- assertTrue(tuples.size() == 10);
- cache.close();
}
-
@Test
- public void testRollupStream() throws Exception {
+ public void testSubFacetStream() throws Exception {
new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
- .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
- .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
- .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
- .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
- .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .add(id, "0", "level1_s", "hello0", "level2_s", "a", "a_i", "0", "a_f", "1")
+ .add(id, "2", "level1_s", "hello0", "level2_s", "a", "a_i", "2", "a_f", "2")
+ .add(id, "3", "level1_s", "hello3", "level2_s", "a", "a_i", "3", "a_f", "3")
+ .add(id, "4", "level1_s", "hello4", "level2_s", "a", "a_i", "4", "a_f", "4")
+ .add(id, "1", "level1_s", "hello0", "level2_s", "b", "a_i", "1", "a_f", "5")
+ .add(id, "5", "level1_s", "hello3", "level2_s", "b", "a_i", "10", "a_f", "6")
+ .add(id, "6", "level1_s", "hello4", "level2_s", "b", "a_i", "11", "a_f", "7")
+ .add(id, "7", "level1_s", "hello3", "level2_s", "b", "a_i", "12", "a_f", "8")
+ .add(id, "8", "level1_s", "hello3", "level2_s", "b", "a_i", "13", "a_f", "9")
+ .add(id, "9", "level1_s", "hello0", "level2_s", "b", "a_i", "14", "a_f", "10")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ String clause;
+ TupleStream stream;
+ List<Tuple> tuples;
+
StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("rollup", RollupStream.class)
+ .withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress())
+ .withFunctionName("facet", FacetStream.class)
.withFunctionName("sum", SumMetric.class)
.withFunctionName("min", MinMetric.class)
.withFunctionName("max", MaxMetric.class)
.withFunctionName("avg", MeanMetric.class)
- .withFunctionName("count", CountMetric.class);
+ .withFunctionName("count", CountMetric.class);
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
- try {
- expression = StreamExpressionParser.parse("rollup("
- + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s asc\"),"
- + "over=\"a_s\","
- + "sum(a_i),"
- + "sum(a_f),"
- + "min(a_i),"
- + "min(a_f),"
- + "max(a_i),"
- + "max(a_f),"
- + "avg(a_i),"
- + "avg(a_f),"
- + "count(*),"
- + ")");
- stream = factory.constructStream(expression);
- stream.setStreamContext(streamContext);
- tuples = getTuples(stream);
-
- assert (tuples.size() == 3);
+ // Basic test
+ clause = "facet("
+ + "collection1, "
+ + "q=\"*:*\", "
+ + "buckets=\"level1_s, level2_s\", "
+ + "bucketSorts=\"sum(a_i) desc, sum(a_i) desc)\", "
+ + "bucketSizeLimit=100, "
+ + "sum(a_i), count(*)"
+ + ")";
+
+ stream = factory.constructStream(clause);
+ tuples = getTuples(stream);
- //Test Long and Double Sums
+ assert(tuples.size() == 6);
- Tuple tuple = tuples.get(0);
- String bucket = tuple.getString("a_s");
- Double sumi = tuple.getDouble("sum(a_i)");
- Double sumf = tuple.getDouble("sum(a_f)");
- Double mini = tuple.getDouble("min(a_i)");
- Double minf = tuple.getDouble("min(a_f)");
- Double maxi = tuple.getDouble("max(a_i)");
- Double maxf = tuple.getDouble("max(a_f)");
- Double avgi = tuple.getDouble("avg(a_i)");
- Double avgf = tuple.getDouble("avg(a_f)");
- Double count = tuple.getDouble("count(*)");
+ Tuple tuple = tuples.get(0);
+ String bucket1 = tuple.getString("level1_s");
+ String bucket2 = tuple.getString("level2_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+ Double count = tuple.getDouble("count(*)");
- assertTrue(bucket.equals("hello0"));
- assertTrue(sumi.doubleValue() == 17.0D);
- assertTrue(sumf.doubleValue() == 18.0D);
- assertTrue(mini.doubleValue() == 0.0D);
- assertTrue(minf.doubleValue() == 1.0D);
- assertTrue(maxi.doubleValue() == 14.0D);
- assertTrue(maxf.doubleValue() == 10.0D);
- assertTrue(avgi.doubleValue() == 4.25D);
- assertTrue(avgf.doubleValue() == 4.5D);
- assertTrue(count.doubleValue() == 4);
+ assertTrue(bucket1.equals("hello3"));
+ assertTrue(bucket2.equals("b"));
+ assertTrue(sumi.longValue() == 35);
+ assertTrue(count.doubleValue() == 3);
- tuple = tuples.get(1);
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
- sumf = tuple.getDouble("sum(a_f)");
- mini = tuple.getDouble("min(a_i)");
- minf = tuple.getDouble("min(a_f)");
- maxi = tuple.getDouble("max(a_i)");
- maxf = tuple.getDouble("max(a_f)");
- avgi = tuple.getDouble("avg(a_i)");
- avgf = tuple.getDouble("avg(a_f)");
- count = tuple.getDouble("count(*)");
+ tuple = tuples.get(1);
+ bucket1 = tuple.getString("level1_s");
+ bucket2 = tuple.getString("level2_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ count = tuple.getDouble("count(*)");
- assertTrue(bucket.equals("hello3"));
- assertTrue(sumi.doubleValue() == 38.0D);
- assertTrue(sumf.doubleValue() == 26.0D);
- assertTrue(mini.doubleValue() == 3.0D);
- assertTrue(minf.doubleValue() == 3.0D);
- assertTrue(maxi.doubleValue() == 13.0D);
- assertTrue(maxf.doubleValue() == 9.0D);
- assertTrue(avgi.doubleValue() == 9.5D);
- assertTrue(avgf.doubleValue() == 6.5D);
- assertTrue(count.doubleValue() == 4);
-
- tuple = tuples.get(2);
- bucket = tuple.getString("a_s");
- sumi = tuple.getDouble("sum(a_i)");
- sumf = tuple.getDouble("sum(a_f)");
- mini = tuple.getDouble("min(a_i)");
- minf = tuple.getDouble("min(a_f)");
- maxi = tuple.getDouble("max(a_i)");
- maxf = tuple.getDouble("max(a_f)");
- avgi = tuple.getDouble("avg(a_i)");
- avgf = tuple.getDouble("avg(a_f)");
- count = tuple.getDouble("count(*)");
+ assertTrue(bucket1.equals("hello0"));
+ assertTrue(bucket2.equals("b"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(count.doubleValue() == 2);
- assertTrue(bucket.equals("hello4"));
- assertTrue(sumi.longValue() == 15);
- assertTrue(sumf.doubleValue() == 11.0D);
- assertTrue(mini.doubleValue() == 4.0D);
- assertTrue(minf.doubleValue() == 4.0D);
- assertTrue(maxi.doubleValue() == 11.0D);
- assertTrue(maxf.doubleValue() == 7.0D);
- assertTrue(avgi.doubleValue() == 7.5D);
- assertTrue(avgf.doubleValue() == 5.5D);
- assertTrue(count.doubleValue() == 2);
+ tuple = tuples.get(2);
+ bucket1 = tuple.getString("level1_s");
+ bucket2 = tuple.getString("level2_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ count = tuple.getDouble("count(*)");
- } finally {
- solrClientCache.close();
- }
- }
+ assertTrue(bucket1.equals("hello4"));
+ assertTrue(bucket2.equals("b"));
+ assertTrue(sumi.longValue() == 11);
+ assertTrue(count.doubleValue() == 1);
- @Test
- public void testStatsStream() throws Exception {
+ tuple = tuples.get(3);
+ bucket1 = tuple.getString("level1_s");
+ bucket2 = tuple.getString("level2_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ count = tuple.getDouble("count(*)");
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
- .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
- .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
- .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
- .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
- .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
- .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("stats", StatsStream.class)
- .withFunctionName("sum", SumMetric.class)
- .withFunctionName("min", MinMetric.class)
- .withFunctionName("max", MaxMetric.class)
- .withFunctionName("avg", MeanMetric.class)
- .withFunctionName("count", CountMetric.class);
-
- StreamExpression expression;
- TupleStream stream;
- List<Tuple> tuples;
- StreamContext streamContext = new StreamContext();
- SolrClientCache cache = new SolrClientCache();
- try {
- streamContext.setSolrClientCache(cache);
- String expr = "stats(" + COLLECTIONORALIAS + ", q=*:*, sum(a_i), sum(a_f), min(a_i), min(a_f), max(a_i), max(a_f), avg(a_i), avg(a_f), count(*))";
- expression = StreamExpressionParser.parse(expr);
- stream = factory.constructStream(expression);
- stream.setStreamContext(streamContext);
-
- tuples = getTuples(stream);
-
- assert (tuples.size() == 1);
-
- //Test Long and Double Sums
-
- Tuple tuple = tuples.get(0);
-
- Double sumi = tuple.getDouble("sum(a_i)");
- Double sumf = tuple.getDouble("sum(a_f)");
- Double mini = tuple.getDouble("min(a_i)");
- Double minf = tuple.getDouble("min(a_f)");
- Double maxi = tuple.getDouble("max(a_i)");
- Double maxf = tuple.getDouble("max(a_f)");
- Double avgi = tuple.getDouble("avg(a_i)");
- Double avgf = tuple.getDouble("avg(a_f)");
- Double count = tuple.getDouble("count(*)");
-
- assertTrue(sumi.longValue() == 70);
- assertTrue(sumf.doubleValue() == 55.0D);
- assertTrue(mini.doubleValue() == 0.0D);
- assertTrue(minf.doubleValue() == 1.0D);
- assertTrue(maxi.doubleValue() == 14.0D);
- assertTrue(maxf.doubleValue() == 10.0D);
- assertTrue(avgi.doubleValue() == 7.0D);
- assertTrue(avgf.doubleValue() == 5.5D);
- assertTrue(count.doubleValue() == 10);
-
-
- //Test with shards parameter
- List<String> shardUrls = TupleStream.getShards(cluster.getZkServer().getZkAddress(), COLLECTIONORALIAS, streamContext);
- expr = "stats(myCollection, q=*:*, sum(a_i), sum(a_f), min(a_i), min(a_f), max(a_i), max(a_f), avg(a_i), avg(a_f), count(*))";
- Map<String, List<String>> shardsMap = new HashMap();
- shardsMap.put("myCollection", shardUrls);
- StreamContext context = new StreamContext();
- context.put("shards", shardsMap);
- context.setSolrClientCache(cache);
- stream = factory.constructStream(expr);
- stream.setStreamContext(context);
-
- tuples = getTuples(stream);
-
- assert (tuples.size() == 1);
-
- //Test Long and Double Sums
-
- tuple = tuples.get(0);
-
- sumi = tuple.getDouble("sum(a_i)");
- sumf = tuple.getDouble("sum(a_f)");
- mini = tuple.getDouble("min(a_i)");
- minf = tuple.getDouble("min(a_f)");
- maxi = tuple.getDouble("max(a_i)");
- maxf = tuple.getDouble("max(a_f)");
- avgi = tuple.getDouble("avg(a_i)");
- avgf = tuple.getDouble("avg(a_f)");
- count = tuple.getDouble("count(*)");
-
- assertTrue(sumi.longValue() == 70);
- assertTrue(sumf.doubleValue() == 55.0D);
- assertTrue(mini.doubleValue() == 0.0D);
- assertTrue(minf.doubleValue() == 1.0D);
- assertTrue(maxi.doubleValue() == 14.0D);
- assertTrue(maxf.doubleValue() == 10.0D);
- assertTrue(avgi.doubleValue() == 7.0D);
- assertTrue(avgf.doubleValue() == 5.5D);
- assertTrue(count.doubleValue() == 10);
-
- //Execersise the /stream hander
-
- //Add the shards http parameter for the myCollection
- StringBuilder buf = new StringBuilder();
- for (String shardUrl : shardUrls) {
- if (buf.length() > 0) {
- buf.append(",");
- }
- buf.append(shardUrl);
- }
-
- ModifiableSolrParams solrParams = new ModifiableSolrParams();
- solrParams.add("qt", "/stream");
- solrParams.add("expr", expr);
- solrParams.add("myCollection.shards", buf.toString());
- SolrStream solrStream = new SolrStream(shardUrls.get(0), solrParams);
- tuples = getTuples(solrStream);
- assert (tuples.size() == 1);
-
- tuple =tuples.get(0);
-
- sumi = tuple.getDouble("sum(a_i)");
- sumf = tuple.getDouble("sum(a_f)");
- mini = tuple.getDouble("min(a_i)");
- minf = tuple.getDouble("min(a_f)");
- maxi = tuple.getDouble("max(a_i)");
- maxf = tuple.getDouble("max(a_f)");
- avgi = tuple.getDouble("avg(a_i)");
- avgf = tuple.getDouble("avg(a_f)");
- count = tuple.getDouble("count(*)");
-
- assertTrue(sumi.longValue() == 70);
- assertTrue(sumf.doubleValue() == 55.0D);
- assertTrue(mini.doubleValue() == 0.0D);
- assertTrue(minf.doubleValue() == 1.0D);
- assertTrue(maxi.doubleValue() == 14.0D);
- assertTrue(maxf.doubleValue() == 10.0D);
- assertTrue(avgi.doubleValue() == 7.0D);
- assertTrue(avgf.doubleValue() == 5.5D);
- assertTrue(count.doubleValue() == 10);
- //Add a negative test to prove that it cannot find slices if shards parameter is removed
-
- try {
- ModifiableSolrParams solrParamsBad = new ModifiableSolrParams();
- solrParamsBad.add("qt", "/stream");
- solrParamsBad.add("expr", expr);
- solrStream = new SolrStream(shardUrls.get(0), solrParamsBad);
- tuples = getTuples(solrStream);
- throw new Exception("Exception should have been thrown above");
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("Collection not found: myCollection"));
- }
- } finally {
- cache.close();
- }
- }
-
- @Test
- public void testParallelUniqueStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1")
- .add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5")
- .add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5")
- .add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- String zkHost = cluster.getZkServer().getZkAddress();
- StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
- .withFunctionName("search", CloudSolrStream.class)
- .withFunctionName("unique", UniqueStream.class)
- .withFunctionName("top", RankStream.class)
- .withFunctionName("group", ReducerStream.class)
- .withFunctionName("parallel", ParallelStream.class);
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
-
-
-
- try {
-
- ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", unique(search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\"), over=\"a_f\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_f asc\")");
- pstream.setStreamContext(streamContext);
- List<Tuple> tuples = getTuples(pstream);
- assert (tuples.size() == 5);
- assertOrder(tuples, 0, 1, 3, 4, 6);
-
- //Test the eofTuples
-
- Map<String, Tuple> eofTuples = pstream.getEofTuples();
- assert (eofTuples.size() == 2); //There should be an EOF tuple for each worker.
- } finally {
- solrClientCache.close();
- }
- }
-
- @Test
- public void testParallelShuffleStream() throws Exception {
-
- new UpdateRequest()
- .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
- .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
- .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
- .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
- .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
- .add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1")
- .add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5")
- .add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5")
- .add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "9", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "10", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "11", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "12", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "13", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "14", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "15", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "16", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "17", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "18", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "19", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "20", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "21", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "22", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "23", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "24", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "25", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "26", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "27", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "28", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "29", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "30", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "31", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "32", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "33", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "34", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "35", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "36", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "37", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "38", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "39", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "40", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "41", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "42", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "43", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "44", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "45", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "46", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "47", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "48", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "49", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "50", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "51", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "52", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "53", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "54", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "55", "a_s", "hello1", "a_i", "13", "a_f", "4")
- .add(id, "56", "a_s", "hello1", "a_i", "13", "a_f", "1000")
-
- .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
-
- StreamContext streamContext = new StreamContext();
- SolrClientCache solrClientCache = new SolrClientCache();
- streamContext.setSolrClientCache(solrClientCache);
-
- String zkHost = cluster.getZkServer().getZkAddress();
- StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
- .withFunctionName("shuffle", ShuffleStream.class)
- .withFunctionName("unique", UniqueStream.class)
- .withFunctionName("parallel", ParallelStream.class);
-
- try {
- ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", unique(shuffle(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\"), over=\"a_f\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_f asc\")");
- pstream.setStreamFactory(streamFactory);
- pstream.setStreamContext(streamContext);
-
<TRUNCATED>
[8/8] lucene-solr:master: SOLR-12183: Fix precommit
Posted by jb...@apache.org.
SOLR-12183: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4137f320
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4137f320
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4137f320
Branch: refs/heads/master
Commit: 4137f320aab4cb69ced9b8da352dd5ad5e1576c3
Parents: d2845b0
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Apr 5 13:24:52 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Apr 5 14:00:08 2018 -0400
----------------------------------------------------------------------
.../apache/solr/client/solrj/io/stream/StreamExpressionTest.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4137f320/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 487aca1..845703e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -22,7 +22,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import org.apache.lucene.util.LuceneTestCase;
[3/8] lucene-solr:master: SOLR-12175: Add random field type and
dynamic field to the default managed-schema
Posted by jb...@apache.org.
SOLR-12175: Add random field type and dynamic field to the default managed-schema
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d420139c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d420139c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d420139c
Branch: refs/heads/master
Commit: d420139c27013ddd8c5aab9dea79dab09736e869
Parents: f1d6911
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue Apr 3 16:25:59 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Apr 5 14:00:07 2018 -0400
----------------------------------------------------------------------
solr/server/solr/configsets/_default/conf/managed-schema | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d420139c/solr/server/solr/configsets/_default/conf/managed-schema
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/_default/conf/managed-schema b/solr/server/solr/configsets/_default/conf/managed-schema
index 4168a83..6f4e2ef 100644
--- a/solr/server/solr/configsets/_default/conf/managed-schema
+++ b/solr/server/solr/configsets/_default/conf/managed-schema
@@ -139,6 +139,7 @@
<dynamicField name="*_fs" type="pfloats" indexed="true" stored="true"/>
<dynamicField name="*_d" type="pdouble" indexed="true" stored="true"/>
<dynamicField name="*_ds" type="pdoubles" indexed="true" stored="true"/>
+ <dynamicField name="random_*" type="random"/>
<!-- Type used for data-driven schema, to add a string copy for each text field -->
<dynamicField name="*_str" type="strings" stored="false" docValues="true" indexed="false" />
@@ -211,6 +212,8 @@
<fieldType name="pfloats" class="solr.FloatPointField" docValues="true" multiValued="true"/>
<fieldType name="plongs" class="solr.LongPointField" docValues="true" multiValued="true"/>
<fieldType name="pdoubles" class="solr.DoublePointField" docValues="true" multiValued="true"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
[2/8] lucene-solr:master: SOLR-12183: Remove dead code
Posted by jb...@apache.org.
SOLR-12183: Remove dead code
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c58516ed
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c58516ed
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c58516ed
Branch: refs/heads/master
Commit: c58516edf1b1525e1341f1427cae066b105c9047
Parents: 80375ac
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Apr 5 12:38:41 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Apr 5 14:00:07 2018 -0400
----------------------------------------------------------------------
.../solrj/io/stream/StreamExpressionTest.java | 52 --------------------
1 file changed, 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c58516ed/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 4d88b4e..487aca1 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -2493,41 +2493,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
return true;
}
- protected boolean assertFields(List<Tuple> tuples, String ... fields) throws Exception{
- for(Tuple tuple : tuples){
- for(String field : fields){
- if(!tuple.fields.containsKey(field)){
- throw new Exception(String.format(Locale.ROOT, "Expected field '%s' not found", field));
- }
- }
- }
- return true;
- }
- protected boolean assertNotFields(List<Tuple> tuples, String ... fields) throws Exception{
- for(Tuple tuple : tuples){
- for(String field : fields){
- if(tuple.fields.containsKey(field)){
- throw new Exception(String.format(Locale.ROOT, "Unexpected field '%s' found", field));
- }
- }
- }
- return true;
- }
-
- protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception {
- List<?> group = (List<?>)tuple.get("tuples");
- int i=0;
- for(int val : ids) {
- Map<?,?> t = (Map<?,?>)group.get(i);
- Long tip = (Long)t.get("id");
- if(tip.intValue() != val) {
- throw new Exception("Found value:"+tip.intValue()+" expecting:"+val);
- }
- ++i;
- }
- return true;
- }
-
public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception {
long lv = (long)tuple.get(fieldName);
if(lv != l) {
@@ -2566,23 +2531,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
return true;
}
- private boolean assertList(List list, Object... vals) throws Exception {
-
- if(list.size() != vals.length) {
- throw new Exception("Lists are not the same size:"+list.size() +" : "+vals.length);
- }
-
- for(int i=0; i<list.size(); i++) {
- Object a = list.get(i);
- Object b = vals[i];
- if(!a.equals(b)) {
- throw new Exception("List items not equals:"+a+" : "+b);
- }
- }
-
- return true;
- }
-
private void assertTopicRun(TupleStream stream, String... idArray) throws Exception {
long version = -1;
int count = 0;
[7/8] lucene-solr:master: SOLR-12183: Refactor Streaming Expression
test cases
Posted by jb...@apache.org.
SOLR-12183: Refactor Streaming Expression test cases
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80375acb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80375acb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80375acb
Branch: refs/heads/master
Commit: 80375acb7f696df7fb3cf0424d5e82777e3f5c87
Parents: d420139
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Apr 5 11:21:49 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Apr 5 14:00:07 2018 -0400
----------------------------------------------------------------------
.../solrj/io/stream/MathExpressionTest.java | 4145 +++++++
.../solrj/io/stream/StreamDecoratorTest.java | 3954 +++++++
.../solrj/io/stream/StreamExpressionTest.java | 10213 ++---------------
3 files changed, 9217 insertions(+), 9095 deletions(-)
----------------------------------------------------------------------
[6/8] lucene-solr:master: SOLR-12183: Refactor Streaming Expression
test cases
Posted by jb...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80375acb/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
new file mode 100644
index 0000000..41116e2
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -0,0 +1,4145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.stream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@Slow
+@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
+public class MathExpressionTest extends SolrCloudTestCase {
+
+ private static final String COLLECTIONORALIAS = "collection1";
+ private static final int TIMEOUT = DEFAULT_TIMEOUT;
+ private static final String id = "id";
+
+ private static boolean useAlias;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(4)
+ .addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
+ .addConfig("ml", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("ml").resolve("conf"))
+ .configure();
+
+ String collection;
+ useAlias = random().nextBoolean();
+ if (useAlias) {
+ collection = COLLECTIONORALIAS + "_collection";
+ } else {
+ collection = COLLECTIONORALIAS;
+ }
+
+ CollectionAdminRequest.createCollection(collection, "conf", 2, 1).process(cluster.getSolrClient());
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(),
+ false, true, TIMEOUT);
+ if (useAlias) {
+ CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection).process(cluster.getSolrClient());
+ }
+ }
+
+ @Before
+ public void cleanIndex() throws Exception {
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ }
+
+ @Test
+ public void testAnalyzeEvaluator() throws Exception {
+
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.add(id, "1", "test_t", "l b c d c");
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+
+ SolrClientCache cache = new SolrClientCache();
+ try {
+
+ String expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id, test_t\", sort=\"id desc\"), analyze(test_t, test_t) as test_t)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+
+ SolrStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 5);
+
+ Tuple t = tuples.get(0);
+ assertTrue(t.getString("test_t").equals("l"));
+ assertTrue(t.getString("id").equals("1"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("test_t").equals("b"));
+ assertTrue(t.getString("id").equals("1"));
+
+
+ t = tuples.get(2);
+ assertTrue(t.getString("test_t").equals("c"));
+ assertTrue(t.getString("id").equals("1"));
+
+
+ t = tuples.get(3);
+ assertTrue(t.getString("test_t").equals("d"));
+ assertTrue(t.getString("id").equals("1"));
+
+ t = tuples.get(4);
+ assertTrue(t.getString("test_t").equals("c"));
+ assertTrue(t.getString("id").equals("1"));
+
+
+ expr = "analyze(\"hello world\", test_t)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertEquals(tuples.size(), 1);
+ List terms = (List)tuples.get(0).get("return-value");
+ assertTrue(terms.get(0).equals("hello"));
+ assertTrue(terms.get(1).equals("world"));
+
+ //Try with single param
+ expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id, test_t\", sort=\"id desc\"), analyze(test_t) as test_t)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 5);
+
+ t = tuples.get(0);
+ assertTrue(t.getString("test_t").equals("l"));
+ assertTrue(t.getString("id").equals("1"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("test_t").equals("b"));
+ assertTrue(t.getString("id").equals("1"));
+
+
+ t = tuples.get(2);
+ assertTrue(t.getString("test_t").equals("c"));
+ assertTrue(t.getString("id").equals("1"));
+
+
+ t = tuples.get(3);
+ assertTrue(t.getString("test_t").equals("d"));
+ assertTrue(t.getString("id").equals("1"));
+
+ t = tuples.get(4);
+ assertTrue(t.getString("test_t").equals("c"));
+ assertTrue(t.getString("id").equals("1"));
+
+ //Try with null in the test_t field
+ expr = "cartesianProduct(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id\", sort=\"id desc\"), analyze(test_t, test_t) as test_t)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+
+ //Test annotating tuple
+ expr = "select(search("+COLLECTIONORALIAS+", q=\"*:*\", fl=\"id, test_t\", sort=\"id desc\"), analyze(test_t, test_t) as test1_t)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List l = (List)tuples.get(0).get("test1_t");
+ assertTrue(l.get(0).equals("l"));
+ assertTrue(l.get(1).equals("b"));
+ assertTrue(l.get(2).equals("c"));
+ assertTrue(l.get(3).equals("d"));
+ assertTrue(l.get(4).equals("c"));
+ } finally {
+ cache.close();
+ }
+ }
+
+ @Test
+ public void testHist() throws Exception {
+ String expr = "hist(sequence(100, 0, 1), 10)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Map> hist = (List<Map>)tuples.get(0).get("return-value");
+ assertTrue(hist.size() == 10);
+ for(int i=0; i<hist.size(); i++) {
+ Map stats = hist.get(i);
+ assertTrue(((Number)stats.get("N")).intValue() == 10);
+ assertTrue(((Number)stats.get("min")).intValue() == 10*i);
+ assertTrue(((Number)stats.get("var")).doubleValue() == 9.166666666666666);
+ assertTrue(((Number)stats.get("stdev")).doubleValue() == 3.0276503540974917);
+ }
+
+ expr = "hist(sequence(100, 0, 1), 5)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ hist = (List<Map>)tuples.get(0).get("return-value");
+ assertTrue(hist.size() == 5);
+ for(int i=0; i<hist.size(); i++) {
+ Map stats = hist.get(i);
+ assertTrue(((Number)stats.get("N")).intValue() == 20);
+ assertTrue(((Number)stats.get("min")).intValue() == 20*i);
+ assertTrue(((Number)stats.get("var")).doubleValue() == 35);
+ assertTrue(((Number)stats.get("stdev")).doubleValue() == 5.916079783099616);
+ }
+ }
+
+ @Test
+ public void testCumulativeProbability() throws Exception {
+ String expr = "cumulativeProbability(normalDistribution(500, 40), 500)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number number = (Number)tuples.get(0).get("return-value");
+ assertTrue(number.doubleValue() == .5D);
+ }
+
+ private String getDateString(String year, String month, String day) {
+ return year+"-"+month+"-"+day+"T00:00:00Z";
+ }
+
+ @Test
+ public void testCorrelationStream() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), " +
+ "tuple(corr=corr(c,d), scorr=corr(array(500, 50, 50, 50),d, type=spearmans), kcorr=corr(array(500, 50, 50, 50),d, type=kendalls), d=d))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ assertTrue(tuples.get(0).getDouble("corr").equals(-1.0D));
+ assertTrue(tuples.get(0).getDouble("scorr").equals(-1.0D));
+ assertTrue(tuples.get(0).getDouble("kcorr").equals(-1.0D));
+ }
+
+ @Test
+ public void testCovariance() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), tuple(colc=c, cold=d, cov=cov(c,d)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ assertTrue(tuples.get(0).getDouble("cov").equals(-625.0D));
+ }
+
+ @Test
+ public void testDistance() throws Exception {
+ String cexpr = "let(echo=true, " +
+ "a=array(1,2,3,4)," +
+ "b=array(2,3,4,5), " +
+ "c=array(3,4,5,6), " +
+ "d=distance(a, b), " +
+ "e=distance(a, c)," +
+ "f=distance(b, c)," +
+ "g=transpose(matrix(a, b, c))," +
+ "h=distance(g)," +
+ "i=distance(a, b, type=manhattan), " +
+ "j=distance(a, c, type=manhattan)," +
+ "k=distance(b, c, type=manhattan)," +
+ "l=transpose(matrix(a, b, c))," +
+ "m=distance(l, type=manhattan)," +
+ "n=distance(a, b, type=canberra), " +
+ "o=distance(a, c, type=canberra)," +
+ "p=distance(b, c, type=canberra)," +
+ "q=transpose(matrix(a, b, c))," +
+ "r=distance(q, type=canberra)," +
+ "s=distance(a, b, type=earthMovers), " +
+ "t=distance(a, c, type=earthMovers)," +
+ "u=distance(b, c, type=earthMovers)," +
+ "w=transpose(matrix(a, b, c))," +
+ "x=distance(w, type=earthMovers)," +
+ ")";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number d = (Number)tuples.get(0).get("d");
+ assertEquals(d.doubleValue(), 2.0, 0.0);
+ Number e = (Number)tuples.get(0).get("e");
+ assertEquals(e.doubleValue(), 4.0, 0.0);
+ Number f = (Number)tuples.get(0).get("f");
+ assertEquals(f.doubleValue(), 2.0, 0.0);
+
+ List<List<Number>> h = (List<List<Number>>)tuples.get(0).get("h");
+ assertEquals(h.size(), 3);
+ assertEquals(h.get(0).size(), 3);
+ List<Number> row0 = h.get(0);
+ assertEquals(row0.get(0).doubleValue(), 0, 0);
+ assertEquals(row0.get(1).doubleValue(), 2, 0);
+ assertEquals(row0.get(2).doubleValue(), 4, 0);
+
+ List<Number> row1 = h.get(1);
+ assertEquals(row1.get(0).doubleValue(), 2, 0);
+ assertEquals(row1.get(1).doubleValue(), 0, 0);
+ assertEquals(row1.get(2).doubleValue(), 2, 0);
+
+ List<Number> row2 = h.get(2);
+ assertEquals(row2.get(0).doubleValue(), 4, 0);
+ assertEquals(row2.get(1).doubleValue(), 2, 0);
+ assertEquals(row2.get(2).doubleValue(), 0, 0);
+
+ Number i = (Number)tuples.get(0).get("i");
+ assertEquals(i.doubleValue(), 4.0, 0.0);
+ Number j = (Number)tuples.get(0).get("j");
+ assertEquals(j.doubleValue(), 8.0, 0.0);
+ Number k = (Number)tuples.get(0).get("k");
+ assertEquals(k.doubleValue(), 4.0, 0.0);
+
+ List<List<Number>> m = (List<List<Number>>)tuples.get(0).get("m");
+ assertEquals(m.size(), 3);
+ assertEquals(m.get(0).size(), 3);
+ row0 = m.get(0);
+ assertEquals(row0.get(0).doubleValue(), 0, 0);
+ assertEquals(row0.get(1).doubleValue(), 4, 0);
+ assertEquals(row0.get(2).doubleValue(), 8, 0);
+
+ row1 = m.get(1);
+ assertEquals(row1.get(0).doubleValue(), 4, 0);
+ assertEquals(row1.get(1).doubleValue(), 0, 0);
+ assertEquals(row1.get(2).doubleValue(), 4, 0);
+
+ row2 = m.get(2);
+ assertEquals(row2.get(0).doubleValue(), 8, 0);
+ assertEquals(row2.get(1).doubleValue(), 4, 0);
+ assertEquals(row2.get(2).doubleValue(), 0, 0);
+
+ Number n = (Number)tuples.get(0).get("n");
+ assertEquals(n.doubleValue(), 0.787302, 0.0001);
+ Number o = (Number)tuples.get(0).get("o");
+ assertEquals(o.doubleValue(), 1.283333, 0.0001);
+ Number p = (Number)tuples.get(0).get("p");
+ assertEquals(p.doubleValue(), 0.544877, 0.0001);
+
+ List<List<Number>> r = (List<List<Number>>)tuples.get(0).get("r");
+ assertEquals(r.size(), 3);
+ assertEquals(r.get(0).size(), 3);
+ row0 = r.get(0);
+ assertEquals(row0.get(0).doubleValue(), 0, 0);
+ assertEquals(row0.get(1).doubleValue(), 0.787302, .0001);
+ assertEquals(row0.get(2).doubleValue(), 1.283333, .0001);
+
+ row1 = r.get(1);
+ assertEquals(row1.get(0).doubleValue(), 0.787302, .0001);
+ assertEquals(row1.get(1).doubleValue(), 0, 0);
+ assertEquals(row1.get(2).doubleValue(), 0.544877, .0001);
+
+ row2 = r.get(2);
+ assertEquals(row2.get(0).doubleValue(), 1.283333, .0001);
+ assertEquals(row2.get(1).doubleValue(), 0.544877, .0001);
+ assertEquals(row2.get(2).doubleValue(), 0, 0);
+
+
+ Number s = (Number)tuples.get(0).get("s");
+ assertEquals(s.doubleValue(), 10.0, 0);
+ Number t = (Number)tuples.get(0).get("t");
+ assertEquals(t.doubleValue(), 20.0, 0);
+ Number u = (Number)tuples.get(0).get("u");
+ assertEquals(u.doubleValue(), 10.0, 0);
+
+ List<List<Number>> x = (List<List<Number>>)tuples.get(0).get("x");
+ assertEquals(x.size(), 3);
+ assertEquals(x.get(0).size(), 3);
+ row0 = x.get(0);
+ assertEquals(row0.get(0).doubleValue(), 0, 0);
+ assertEquals(row0.get(1).doubleValue(), 10.0, 0);
+ assertEquals(row0.get(2).doubleValue(), 20, 0);
+
+ row1 = x.get(1);
+ assertEquals(row1.get(0).doubleValue(), 10, 0);
+ assertEquals(row1.get(1).doubleValue(), 0, 0);
+ assertEquals(row1.get(2).doubleValue(), 10, 0);
+
+ row2 = x.get(2);
+ assertEquals(row2.get(0).doubleValue(), 20, 0);
+ assertEquals(row2.get(1).doubleValue(), 10, 0);
+ assertEquals(row2.get(2).doubleValue(), 0, 0);
+ }
+
+ @Test
+ public void testReverse() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(reverse=rev(c)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> reverse = (List<Number>)tuples.get(0).get("reverse");
+ assertTrue(reverse.size() == 4);
+ assertTrue(reverse.get(0).doubleValue() == 400D);
+ assertTrue(reverse.get(1).doubleValue() == 300D);
+ assertTrue(reverse.get(2).doubleValue() == 500D);
+ assertTrue(reverse.get(3).doubleValue() == 100D);
+ }
+
+ @Test
+ public void testCopyOf() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(copy1=copyOf(c, 10), copy2=copyOf(c), copy3=copyOf(c, 2) ))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> copy1 = (List<Number>)tuples.get(0).get("copy1");
+ assertTrue(copy1.size() == 4);
+ assertTrue(copy1.get(0).doubleValue() == 100D);
+ assertTrue(copy1.get(1).doubleValue() == 500D);
+ assertTrue(copy1.get(2).doubleValue() == 300D);
+ assertTrue(copy1.get(3).doubleValue() == 400D);
+
+ List<Number> copy2 = (List<Number>)tuples.get(0).get("copy2");
+ assertTrue(copy2.size() == 4);
+ assertTrue(copy2.get(0).doubleValue() == 100D);
+ assertTrue(copy2.get(1).doubleValue() == 500D);
+ assertTrue(copy2.get(2).doubleValue() == 300D);
+ assertTrue(copy2.get(3).doubleValue() == 400D);
+
+ List<Number> copy3 = (List<Number>)tuples.get(0).get("copy3");
+ assertTrue(copy3.size() == 2);
+ assertTrue(copy3.get(0).doubleValue() == 100D);
+ assertTrue(copy3.get(1).doubleValue() == 500D);
+ }
+
+ @Test
+ public void testCopyOfRange() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(copy=copyOfRange(c, 1, 3), copy2=copyOfRange(c, 2, 4), l=length(c)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> copy1 = (List<Number>)tuples.get(0).get("copy");
+ assertTrue(copy1.size() == 2);
+ assertTrue(copy1.get(0).doubleValue() == 500D);
+ assertTrue(copy1.get(1).doubleValue() == 300D);
+
+ List<Number> copy2 = (List<Number>)tuples.get(0).get("copy2");
+ assertTrue(copy2.size() == 2);
+ assertTrue(copy2.get(0).doubleValue() == 300D);
+ assertTrue(copy2.get(1).doubleValue() == 400D);
+
+ long l = tuples.get(0).getLong("l");
+ assertTrue(l == 4);
+
+ }
+
+ @Test
+ public void testPercentile() throws Exception {
+ String cexpr = "percentile(array(1,2,3,4,5,6,7,8,9,10,11), 50)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ double p = tuple.getDouble("return-value");
+ assertEquals(p, 6, 0.0);
+
+
+ cexpr = "percentile(array(11,10,3,4,5,6,7,8,9,2,1), 50)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ tuple = tuples.get(0);
+ p = tuple.getDouble("return-value");
+ assertEquals(p, 6, 0.0);
+
+ cexpr = "percentile(array(11,10,3,4,5,6,7,8,9,2,1), 20)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ tuple = tuples.get(0);
+ p = tuple.getDouble("return-value");
+ assertEquals(p, 2.4, 0.001);
+ }
+
+ @Test
+ public void testPrimes() throws Exception {
+ String cexpr = "primes(10, 0)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ List<Number> asort = (List<Number>)tuple.get("return-value");
+ assertEquals(asort.size(), 10);
+ assertEquals(asort.get(0).intValue(), 2);
+ assertEquals(asort.get(1).intValue(), 3);
+ assertEquals(asort.get(2).intValue(), 5);
+ assertEquals(asort.get(3).intValue(), 7);
+ assertEquals(asort.get(4).intValue(), 11);
+ assertEquals(asort.get(5).intValue(), 13);
+ assertEquals(asort.get(6).intValue(), 17);
+ assertEquals(asort.get(7).intValue(), 19);
+ assertEquals(asort.get(8).intValue(), 23);
+ assertEquals(asort.get(9).intValue(), 29);
+ }
+
+ @Test
+ public void testBinomialCoefficient() throws Exception {
+ String cexpr = "binomialCoefficient(8,3)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ long binomialCoefficient = (long) tuple.get("return-value");
+ assertEquals(binomialCoefficient, 56);
+ }
+
+ @Test
+ public void testAscend() throws Exception {
+ String cexpr = "asc(array(11.5, 12.3, 4, 3, 1, 0))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ List<Number> asort = (List<Number>)tuple.get("return-value");
+ assertEquals(asort.size(), 6);
+ assertEquals(asort.get(0).doubleValue(), 0, 0.0);
+ assertEquals(asort.get(1).doubleValue(), 1, 0.0);
+ assertEquals(asort.get(2).doubleValue(), 3, 0.0);
+ assertEquals(asort.get(3).doubleValue(), 4, 0.0);
+ assertEquals(asort.get(4).doubleValue(), 11.5, 0.0);
+ assertEquals(asort.get(5).doubleValue(), 12.3, 0.0);
+ }
+
+ @Test
+ public void testRankTransform() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<50) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+ }
+
+ while(i<100) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2015", "5", "1"), "price_f", "300.0");
+ }
+
+ while(i<150) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2014", "5", "1"), "price_f", "500.0");
+ }
+
+ while(i<250) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2013", "5", "1"), "price_f", "100.00");
+ }
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+
+ String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(reverse=rev(c), ranked=rank(c)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> reverse = (List<Number>)tuples.get(0).get("reverse");
+ assertTrue(reverse.size() == 4);
+ assertTrue(reverse.get(0).doubleValue() == 400D);
+ assertTrue(reverse.get(1).doubleValue() == 300D);
+ assertTrue(reverse.get(2).doubleValue() == 500D);
+ assertTrue(reverse.get(3).doubleValue() == 100D);
+
+ List<Number> ranked = (List<Number>)tuples.get(0).get("ranked");
+ assertTrue(ranked.size() == 4);
+ assertTrue(ranked.get(0).doubleValue() == 1D);
+ assertTrue(ranked.get(1).doubleValue() == 4D);
+ assertTrue(ranked.get(2).doubleValue() == 2D);
+ assertTrue(ranked.get(3).doubleValue() == 3D);
+ }
+
+ @Test
+ public void testArray() throws Exception {
+ String cexpr = "array(1, 2, 3, 300, 2, 500)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>)tuples.get(0).get("return-value");
+ assertTrue(out.size() == 6);
+ assertTrue(out.get(0).intValue() == 1);
+ assertTrue(out.get(1).intValue() == 2);
+ assertTrue(out.get(2).intValue() == 3);
+ assertTrue(out.get(3).intValue() == 300);
+ assertTrue(out.get(4).intValue() == 2);
+ assertTrue(out.get(5).intValue() == 500);
+
+ cexpr = "array(1.122, 2.222, 3.333, 300.1, 2.13, 500.23)";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ out = (List<Number>)tuples.get(0).get("return-value");
+ assertTrue(out.size() == 6);
+ assertTrue(out.get(0).doubleValue() == 1.122D);
+ assertTrue(out.get(1).doubleValue() == 2.222D);
+ assertTrue(out.get(2).doubleValue() == 3.333D);
+ assertTrue(out.get(3).doubleValue() == 300.1D);
+ assertTrue(out.get(4).doubleValue() == 2.13D);
+ assertTrue(out.get(5).doubleValue() == 500.23D);
+ }
+
+ @Test
+ public void testMatrix() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=setColumnLabels(matrix(array(1, 2, 3), " +
+ " rev(array(4,5,6)))," +
+ " array(col1, col2, col3))," +
+ " b=rowAt(a, 1)," +
+ " c=colAt(a, 2)," +
+ " d=getColumnLabels(a)," +
+ " e=topFeatures(a, 1)," +
+ " f=rowCount(a)," +
+ " g=columnCount(a)," +
+ " h=indexOf(d, \"col2\")," +
+ " i=indexOf(d, col3))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> out = (List<List<Number>>)tuples.get(0).get("a");
+
+ List<Number> array1 = out.get(0);
+ assertEquals(array1.size(), 3);
+ assertEquals(array1.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(array1.get(1).doubleValue(), 2.0, 0.0);
+ assertEquals(array1.get(2).doubleValue(), 3.0, 0.0);
+
+ List<Number> array2 = out.get(1);
+ assertEquals(array2.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), 6.0, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 5.0, 0.0);
+ assertEquals(array2.get(2).doubleValue(), 4.0, 0.0);
+
+ List<Number> row = (List<Number>)tuples.get(0).get("b");
+
+ assertEquals(row.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), 6.0, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 5.0, 0.0);
+ assertEquals(array2.get(2).doubleValue(), 4.0, 0.0);
+
+ List<Number> col = (List<Number>)tuples.get(0).get("c");
+ assertEquals(col.size(), 2);
+ assertEquals(col.get(0).doubleValue(), 3.0, 0.0);
+ assertEquals(col.get(1).doubleValue(), 4.0, 0.0);
+
+ List<String> colLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(colLabels.size(), 3);
+ assertEquals(colLabels.get(0), "col1");
+ assertEquals(colLabels.get(1), "col2");
+ assertEquals(colLabels.get(2), "col3");
+
+ List<List<String>> features = (List<List<String>>)tuples.get(0).get("e");
+ assertEquals(features.size(), 2);
+ assertEquals(features.get(0).size(), 1);
+ assertEquals(features.get(1).size(), 1);
+ assertEquals(features.get(0).get(0), "col3");
+ assertEquals(features.get(1).get(0), "col1");
+
+ assertTrue(tuples.get(0).getLong("f") == 2);
+ assertTrue(tuples.get(0).getLong("g")== 3);
+ assertTrue(tuples.get(0).getLong("h")== 1);
+ assertTrue(tuples.get(0).getLong("i")== 2);
+ }
+
+ @Test
+ public void testMatrixMath() throws Exception {
+ String cexpr = "let(echo=true, a=matrix(array(1.5, 2.5, 3.5), array(4.5,5.5,6.5)), " +
+ "b=grandSum(a), " +
+ "c=sumRows(a), " +
+ "d=sumColumns(a), " +
+ "e=scalarAdd(1, a)," +
+ "f=scalarSubtract(1, a)," +
+ "g=scalarMultiply(1.5, a)," +
+ "h=scalarDivide(1.5, a)," +
+ "i=scalarAdd(1.5, array(1.5, 2.5, 3.5))," +
+ "j=scalarSubtract(1.5, array(1.5, 2.5, 3.5))," +
+ "k=scalarMultiply(1.5, array(1.5, 2.5, 3.5))," +
+ "l=scalarDivide(1.5, array(1.5, 2.5, 3.5)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+
+ double grandSum = tuples.get(0).getDouble("b");
+ assertEquals(grandSum, 24, 0.0);
+
+ List<Number> sumRows = (List<Number>)tuples.get(0).get("c");
+ assertEquals(sumRows.size(), 2);
+ assertEquals(sumRows.get(0).doubleValue(), 7.5, 0.0);
+ assertEquals(sumRows.get(1).doubleValue(), 16.5, 0.0);
+
+ List<Number> sumCols = (List<Number>)tuples.get(0).get("d");
+ assertEquals(sumCols.size(), 3);
+ assertEquals(sumCols.get(0).doubleValue(), 6.0, 0.0);
+ assertEquals(sumCols.get(1).doubleValue(), 8.0, 0.0);
+ assertEquals(sumCols.get(2).doubleValue(), 10, 0.0);
+
+ List<List<Number>> scalarAdd = (List<List<Number>>)tuples.get(0).get("e");
+ List<Number> row1 = scalarAdd.get(0);
+ assertEquals(row1.size(), 3);
+ assertEquals(row1.get(0).doubleValue(), 2.5, 0.0);
+ assertEquals(row1.get(1).doubleValue(), 3.5, 0.0);
+ assertEquals(row1.get(2).doubleValue(), 4.5, 0.0);
+
+ List<Number> row2 = scalarAdd.get(1);
+ assertEquals(row2.get(0).doubleValue(), 5.5, 0.0);
+ assertEquals(row2.get(1).doubleValue(), 6.5, 0.0);
+ assertEquals(row2.get(2).doubleValue(), 7.5, 0.0);
+
+ List<List<Number>> scalarSubtract = (List<List<Number>>)tuples.get(0).get("f");
+ row1 = scalarSubtract.get(0);
+ assertEquals(row1.size(), 3);
+ assertEquals(row1.get(0).doubleValue(), 0.5, 0.0);
+ assertEquals(row1.get(1).doubleValue(), 1.5, 0.0);
+ assertEquals(row1.get(2).doubleValue(), 2.5, 0.0);
+
+ row2 = scalarSubtract.get(1);
+ assertEquals(row2.get(0).doubleValue(), 3.5, 0.0);
+ assertEquals(row2.get(1).doubleValue(), 4.5, 0.0);
+ assertEquals(row2.get(2).doubleValue(), 5.5, 0.0);
+
+ List<List<Number>> scalarMultiply = (List<List<Number>>)tuples.get(0).get("g");
+ row1 = scalarMultiply.get(0);
+ assertEquals(row1.size(), 3);
+ assertEquals(row1.get(0).doubleValue(), 2.25, 0.0);
+ assertEquals(row1.get(1).doubleValue(), 3.75, 0.0);
+ assertEquals(row1.get(2).doubleValue(), 5.25, 0.0);
+
+ row2 = scalarMultiply.get(1);
+ assertEquals(row2.get(0).doubleValue(), 6.75, 0.0);
+ assertEquals(row2.get(1).doubleValue(), 8.25, 0.0);
+ assertEquals(row2.get(2).doubleValue(), 9.75, 0.0);
+
+ List<List<Number>> scalarDivide = (List<List<Number>>)tuples.get(0).get("h");
+ row1 = scalarDivide.get(0);
+ assertEquals(row1.size(), 3);
+ assertEquals(row1.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(row1.get(1).doubleValue(), 1.66666666666667, 0.001);
+ assertEquals(row1.get(2).doubleValue(), 2.33333333333333, 0.001);
+
+ row2 = scalarDivide.get(1);
+ assertEquals(row2.get(0).doubleValue(), 3, 0.0);
+ assertEquals(row2.get(1).doubleValue(), 3.66666666666667, 0.001);
+ assertEquals(row2.get(2).doubleValue(), 4.33333333333333, 0.001);
+
+ List<Number> rowA = (List<Number>)tuples.get(0).get("i");
+ assertEquals(rowA.size(), 3);
+ assertEquals(rowA.get(0).doubleValue(), 3.0, 0.0);
+ assertEquals(rowA.get(1).doubleValue(), 4.0, 0.0);
+ assertEquals(rowA.get(2).doubleValue(), 5.0, 0.0);
+
+ rowA = (List<Number>)tuples.get(0).get("j");
+ assertEquals(rowA.size(), 3);
+ assertEquals(rowA.get(0).doubleValue(), 0, 0.0);
+ assertEquals(rowA.get(1).doubleValue(), 1.0, 0.0);
+ assertEquals(rowA.get(2).doubleValue(), 2.0, 0.0);
+
+ rowA = (List<Number>)tuples.get(0).get("k");
+ assertEquals(rowA.size(), 3);
+ assertEquals(rowA.get(0).doubleValue(), 2.25, 0.0);
+ assertEquals(rowA.get(1).doubleValue(), 3.75, 0.0);
+ assertEquals(rowA.get(2).doubleValue(), 5.25, 0.0);
+
+ rowA = (List<Number>)tuples.get(0).get("l");
+ assertEquals(rowA.size(), 3);
+ assertEquals(rowA.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(rowA.get(1).doubleValue(), 1.66666666666667, 0.001);
+ assertEquals(rowA.get(2).doubleValue(), 2.33333333333333, 0.001);
+ }
+
+ @Test
+ public void testTranspose() throws Exception {
+ String cexpr = "let(a=matrix(array(1,2,3), array(4,5,6)), b=transpose(a))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> out = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(out.size(), 3);
+ List<Number> array1 = out.get(0);
+ assertEquals(array1.size(), 2);
+ assertEquals(array1.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(array1.get(1).doubleValue(), 4.0, 0.0);
+
+ List<Number> array2 = out.get(1);
+ assertEquals(array2.size(), 2);
+ assertEquals(array2.get(0).doubleValue(), 2.0, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 5.0, 0.0);
+
+ List<Number> array3 = out.get(2);
+ assertEquals(array3.size(), 2);
+ assertEquals(array3.get(0).doubleValue(), 3.0, 0.0);
+ assertEquals(array3.get(1).doubleValue(), 6.0, 0.0);
+ }
+
+ @Test
+ public void testUnitize() throws Exception {
+ String cexpr = "let(echo=true, a=unitize(matrix(array(1,2,3), array(4,5,6))), b=unitize(array(4,5,6)))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> out = (List<List<Number>>)tuples.get(0).get("a");
+ assertEquals(out.size(), 2);
+ List<Number> array1 = out.get(0);
+ assertEquals(array1.size(), 3);
+ assertEquals(array1.get(0).doubleValue(), 0.2672612419124244, 0.0);
+ assertEquals(array1.get(1).doubleValue(), 0.5345224838248488, 0.0);
+ assertEquals(array1.get(2).doubleValue(), 0.8017837257372732, 0.0);
+
+ List<Number> array2 = out.get(1);
+ assertEquals(array2.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), 0.4558423058385518, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 0.5698028822981898, 0.0);
+ assertEquals(array2.get(2).doubleValue(), 0.6837634587578276, 0.0);
+
+ List<Number> array3 = (List<Number>)tuples.get(0).get("b");
+ assertEquals(array3.size(), 3);
+ assertEquals(array3.get(0).doubleValue(), 0.4558423058385518, 0.0);
+ assertEquals(array3.get(1).doubleValue(), 0.5698028822981898, 0.0);
+ assertEquals(array3.get(2).doubleValue(), 0.6837634587578276, 0.0);
+ }
+
+ @Test
+ public void testNormalizeSum() throws Exception {
+ String cexpr = "let(echo=true, " +
+ "a=normalizeSum(matrix(array(1,2,3), array(4,5,6))), " +
+ "b=normalizeSum(array(1,2,3))," +
+ "c=normalizeSum(array(1,2,3), 100))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> out = (List<List<Number>>)tuples.get(0).get("a");
+ assertEquals(out.size(), 2);
+ List<Number> array1 = out.get(0);
+ assertEquals(array1.size(), 3);
+ assertEquals(array1.get(0).doubleValue(), 0.16666666666666666, 0.0001);
+ assertEquals(array1.get(1).doubleValue(), 0.3333333333333333, 0.00001);
+ assertEquals(array1.get(2).doubleValue(), 0.5, 0.0001);
+
+ List<Number> array2 = out.get(1);
+ assertEquals(array2.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), 0.26666666666666666, 0.0001);
+ assertEquals(array2.get(1).doubleValue(), 0.3333333333333333, 0.0001);
+ assertEquals(array2.get(2).doubleValue(), 0.4, 0.0001);
+
+ List<Number> array3 = (List<Number>)tuples.get(0).get("b");
+ assertEquals(array3.size(), 3);
+ assertEquals(array3.get(0).doubleValue(), 0.16666666666666666, 0.0001);
+ assertEquals(array3.get(1).doubleValue(), 0.3333333333333333, 0.0001);
+ assertEquals(array3.get(2).doubleValue(), 0.5, 0.0001);
+
+ List<Number> array4 = (List<Number>)tuples.get(0).get("c");
+ assertEquals(array4.size(), 3);
+ assertEquals(array4.get(0).doubleValue(), 16.666666666666666, 0.0001);
+ assertEquals(array4.get(1).doubleValue(), 33.33333333333333, 0.00001);
+ assertEquals(array4.get(2).doubleValue(), 50, 0.0001);
+ }
+
+ @Test
+ public void testStandardize() throws Exception {
+ String cexpr = "let(echo=true, a=standardize(matrix(array(1,2,3), array(4,5,6))), b=standardize(array(4,5,6)))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> out = (List<List<Number>>)tuples.get(0).get("a");
+ assertEquals(out.size(), 2);
+ List<Number> array1 = out.get(0);
+ assertEquals(array1.size(), 3);
+ assertEquals(array1.get(0).doubleValue(), -1, 0.0);
+ assertEquals(array1.get(1).doubleValue(), 0, 0.0);
+ assertEquals(array1.get(2).doubleValue(), 1, 0.0);
+
+ List<Number> array2 = out.get(1);
+ assertEquals(array2.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), -1, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 0, 0.0);
+ assertEquals(array2.get(2).doubleValue(), 1, 0.0);
+
+ List<Number> array3 = (List<Number>)tuples.get(0).get("b");
+ assertEquals(array3.size(), 3);
+ assertEquals(array2.get(0).doubleValue(), -1, 0.0);
+ assertEquals(array2.get(1).doubleValue(), 0, 0.0);
+ assertEquals(array2.get(2).doubleValue(), 1, 0.0);
+ }
+
+ @Test
+ public void testMarkovChain() throws Exception {
+ String cexpr = "let(state0=array(.5,.5),\n" +
+ " state1=array(.5,.5),\n" +
+ " states=matrix(state0, state1),\n" +
+ " m=markovChain(states, 0),\n" +
+ " s=sample(m, 50000),\n" +
+ " f=freqTable(s))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Map<String, Number>> out = (List<Map<String, Number>>)tuples.get(0).get("f");
+ assertEquals(out.size(), 2);
+ Map<String, Number> bin0 = out.get(0);
+ double state0Pct = bin0.get("pct").doubleValue();
+ assertEquals(state0Pct, .5, .015);
+ Map<String, Number> bin1 = out.get(1);
+ double state1Pct = bin1.get("pct").doubleValue();
+ assertEquals(state1Pct, .5, .015);
+ }
+
+ @Test
+ public void testAddAll() throws Exception {
+ String cexpr = "addAll(array(1, 2, 3), array(4.5, 5.5, 6.5), array(7,8,9))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>)tuples.get(0).get("return-value");
+ assertTrue(out.size() == 9);
+ assertTrue(out.get(0).intValue() == 1);
+ assertTrue(out.get(1).intValue() == 2);
+ assertTrue(out.get(2).intValue() == 3);
+ assertTrue(out.get(3).doubleValue() == 4.5D);
+ assertTrue(out.get(4).doubleValue() == 5.5D);
+ assertTrue(out.get(5).doubleValue() == 6.5D);
+ assertTrue(out.get(6).intValue() == 7);
+ assertTrue(out.get(7).intValue() == 8);
+ assertTrue(out.get(8).intValue() == 9);
+ }
+
+ @Test
+ public void testProbabilityRange() throws Exception {
+ String cexpr = "let(a=normalDistribution(500, 20), " +
+ "b=probability(a, 520, 530))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number prob = (Number)tuples.get(0).get("b");
+ assertEquals(prob.doubleValue(), 0.09184805266259899, 0.0);
+ }
+
+ @Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+ public void testDistributions() throws Exception {
+ String cexpr = "let(a=normalDistribution(10, 2), " +
+ "b=sample(a, 250), " +
+ "c=normalDistribution(100, 6), " +
+ "d=sample(c, 250), " +
+ "u=uniformDistribution(1, 6),"+
+ "t=sample(u, 250),"+
+ "e=empiricalDistribution(d),"+
+ "f=sample(e, 250),"+
+ "tuple(sample=b, ks=ks(a,b), ks2=ks(a, d), ks3=ks(u, t)))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ try {
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>) tuples.get(0).get("sample");
+
+ Map ks = (Map) tuples.get(0).get("ks");
+ Map ks2 = (Map) tuples.get(0).get("ks2");
+ Map ks3 = (Map) tuples.get(0).get("ks3");
+
+ assertTrue(out.size() == 250);
+ Number pvalue = (Number) ks.get("p-value");
+ Number pvalue2 = (Number) ks2.get("p-value");
+ Number pvalue3 = (Number) ks3.get("p-value");
+
+ assertTrue(pvalue.doubleValue() > .05D);
+ assertTrue(pvalue2.doubleValue() == 0);
+ assertTrue(pvalue3.doubleValue() > .05D);
+
+ } catch(AssertionError e) {
+
+ //This test will have random failures do to the random sampling. So if it fails try it again.
+ //If it fails twice in a row, we probably broke some code.
+
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>) tuples.get(0).get("sample");
+
+ Map ks = (Map) tuples.get(0).get("ks");
+ Map ks2 = (Map) tuples.get(0).get("ks2");
+ Map ks3 = (Map) tuples.get(0).get("ks3");
+
+ assertTrue(out.size() == 250);
+ Number pvalue = (Number) ks.get("p-value");
+ Number pvalue2 = (Number) ks2.get("p-value");
+ Number pvalue3 = (Number) ks3.get("p-value");
+
+ assertTrue(pvalue.doubleValue() > .05D);
+ assertTrue(pvalue2.doubleValue() == 0);
+ assertTrue(pvalue3.doubleValue() > .05D);
+ }
+ }
+
+ @Test
+ public void testSumDifference() throws Exception {
+ String cexpr = "sumDifference(array(2,4,6,8,10,12),array(1,2,3,4,5,6))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ double sd = tuples.get(0).getDouble("return-value");
+ assertEquals(sd, 21.0D, 0.0);
+ }
+
+ @Test
+ public void testMeanDifference() throws Exception {
+ String cexpr = "meanDifference(array(2,4,6,8,10,12),array(1,2,3,4,5,6))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ double sd = tuples.get(0).getDouble("return-value");
+ assertEquals(sd, 3.5, 0.0);
+ }
+
+ @Test
+ public void testTermVectors() throws Exception {
+ // Test termVectors with only documents and default termVector settings
+ String cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minDocFreq=0, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> termVectors = (List<List<Number>>)tuples.get(0).get("b");
+
+ assertEquals(termVectors.size(), 4);
+ List<Number> termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 1.916290731874155, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 2.7100443424662948, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 5);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(4).doubleValue(), 0.0, 0.0);
+
+ List<String> rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ List<String> columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 5);
+ assertEquals(columnLabels.get(0), "hello");
+ assertEquals(columnLabels.get(1), "jack");
+ assertEquals(columnLabels.get(2), "jim");
+ assertEquals(columnLabels.get(3), "steve");
+ assertEquals(columnLabels.get(4), "world");
+
+ Map<String, Number> docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 5);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+ assertEquals(docFreqs.get("jack").intValue(), 1);
+ assertEquals(docFreqs.get("jim").intValue(), 1);
+ assertEquals(docFreqs.get("steve").intValue(), 1);
+ assertEquals(docFreqs.get("world").intValue(), 1);
+
+ //Test minTermLength. This should drop off the term jim
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minTermLength=4, minDocFreq=0, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(termVectors.size(), 4);
+ termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 1.916290731874155, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 4);
+ assertEquals(columnLabels.get(0), "hello");
+ assertEquals(columnLabels.get(1), "jack");
+ assertEquals(columnLabels.get(2), "steve");
+ assertEquals(columnLabels.get(3), "world");
+
+ docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 4);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+ assertEquals(docFreqs.get("jack").intValue(), 1);
+ assertEquals(docFreqs.get("steve").intValue(), 1);
+ assertEquals(docFreqs.get("world").intValue(), 1);
+
+
+ //Test exclude. This should drop off the term jim
+
+ cexpr = "let(echo=true," +
+ " a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ " tuple(id=\"2\", text=\"hello steve\"), " +
+ " tuple(id=\"3\", text=\"hello jim jim\"), " +
+ " tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, exclude=jim, minDocFreq=0, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(termVectors.size(), 4);
+ termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 1.916290731874155, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 4);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(termVector.get(1).doubleValue(), 1.916290731874155, 0.0);
+ assertEquals(termVector.get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(termVector.get(3).doubleValue(), 0.0, 0.0);
+
+ rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 4);
+ assertEquals(columnLabels.get(0), "hello");
+ assertEquals(columnLabels.get(1), "jack");
+ assertEquals(columnLabels.get(2), "steve");
+ assertEquals(columnLabels.get(3), "world");
+
+ docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 4);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+ assertEquals(docFreqs.get("jack").intValue(), 1);
+ assertEquals(docFreqs.get("steve").intValue(), 1);
+ assertEquals(docFreqs.get("world").intValue(), 1);
+
+ //Test minDocFreq attribute at .5. This should eliminate all but the term hello
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, minDocFreq=.5, maxDocFreq=1)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+
+ assertEquals(termVectors.size(), 4);
+ termVector = termVectors.get(0);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(1);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(2);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ termVector = termVectors.get(3);
+ assertEquals(termVector.size(), 1);
+ assertEquals(termVector.get(0).doubleValue(), 1.0, 0.0);
+
+ rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.size(), 4);
+ assertEquals(rowLabels.get(0), "1");
+ assertEquals(rowLabels.get(1), "2");
+ assertEquals(rowLabels.get(2), "3");
+ assertEquals(rowLabels.get(3), "4");
+
+ columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.size(), 1);
+ assertEquals(columnLabels.get(0), "hello");
+
+ docFreqs = (Map<String, Number>)tuples.get(0).get("e");
+
+ assertEquals(docFreqs.size(), 1);
+ assertEquals(docFreqs.get("hello").intValue(), 4);
+
+ //Test maxDocFreq attribute at 0. This should eliminate all terms
+
+ cexpr = "let(echo=true," +
+ "a=select(list(tuple(id=\"1\", text=\"hello world\"), " +
+ "tuple(id=\"2\", text=\"hello steve\"), " +
+ "tuple(id=\"3\", text=\"hello jim jim\"), " +
+ "tuple(id=\"4\", text=\"hello jack\")), id, analyze(text, test_t) as terms)," +
+ " b=termVectors(a, maxDocFreq=0)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b)," +
+ " e=getAttribute(b, docFreqs))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ termVectors = (List<List<Number>>)tuples.get(0).get("b");
+ assertEquals(termVectors.size(), 4);
+ assertEquals(termVectors.get(0).size(), 0);
+ }
+
+ @Test
+ public void testEbeSubtract() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(2, 4, 6, 8, 10, 12)," +
+ " b=array(1, 2, 3, 4, 5, 6)," +
+ " c=ebeSubtract(a,b)," +
+ " d=array(10, 11, 12, 13, 14, 15)," +
+ " e=array(100, 200, 300, 400, 500, 600)," +
+ " f=matrix(a, b)," +
+ " g=matrix(d, e)," +
+ " h=ebeSubtract(f, g))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>)tuples.get(0).get("c");
+ assertEquals(out.size(), 6);
+ assertEquals(out.get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(out.get(1).doubleValue(), 2.0, 0.0);
+ assertEquals(out.get(2).doubleValue(), 3.0, 0.0);
+ assertEquals(out.get(3).doubleValue(), 4.0, 0.0);
+ assertEquals(out.get(4).doubleValue(), 5.0, 0.0);
+ assertEquals(out.get(5).doubleValue(), 6.0, 0.0);
+
+ List<List<Number>> mout = (List<List<Number>>)tuples.get(0).get("h");
+ assertEquals(mout.size(), 2);
+ List<Number> row1 = mout.get(0);
+ assertEquals(row1.size(), 6);
+ assertEquals(row1.get(0).doubleValue(), -8.0, 0.0);
+ assertEquals(row1.get(1).doubleValue(), -7.0, 0.0);
+ assertEquals(row1.get(2).doubleValue(), -6.0, 0.0);
+ assertEquals(row1.get(3).doubleValue(), -5.0, 0.0);
+ assertEquals(row1.get(4).doubleValue(), -4.0, 0.0);
+ assertEquals(row1.get(5).doubleValue(), -3.0, 0.0);
+
+ List<Number> row2 = mout.get(1);
+ assertEquals(row2.size(), 6);
+ assertEquals(row2.get(0).doubleValue(), -99.0, 0.0);
+ assertEquals(row2.get(1).doubleValue(), -198.0, 0.0);
+ assertEquals(row2.get(2).doubleValue(), -297.0, 0.0);
+ assertEquals(row2.get(3).doubleValue(), -396.0, 0.0);
+ assertEquals(row2.get(4).doubleValue(), -495.0, 0.0);
+ assertEquals(row2.get(5).doubleValue(), -594.0, 0.0);
+ }
+
+ @Test
+ public void testMatrixMult() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(1,2,3)," +
+ " b=matrix(array(4), array(5), array(6))," +
+ " c=matrixMult(a, b)," +
+ " d=matrix(array(3, 4), array(10,11), array(30, 40))," +
+ " e=matrixMult(a, d)," +
+ " f=array(4,8,10)," +
+ " g=matrix(a, f)," +
+ " h=matrixMult(d, g)," +
+ " i=matrixMult(b, a))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> matrix = (List<List<Number>>)tuples.get(0).get("c");
+ assertEquals(matrix.size(), 1);
+ List<Number> row = matrix.get(0);
+ assertEquals(row.size(), 1);
+ assertEquals(row.get(0).doubleValue(), 32.0, 0.0);
+
+ matrix = (List<List<Number>>)tuples.get(0).get("e");
+ assertEquals(matrix.size(), 1);
+ row = matrix.get(0);
+ assertEquals(row.size(), 2);
+ assertEquals(row.get(0).doubleValue(), 113.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 146.0, 0.0);
+
+ matrix = (List<List<Number>>)tuples.get(0).get("h");
+ assertEquals(matrix.size(), 3);
+ row = matrix.get(0);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 19.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 38.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 49.0, 0.0);
+
+ row = matrix.get(1);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 54.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 108.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 140.0, 0.0);
+
+ row = matrix.get(2);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 190.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 380.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 490.0, 0.0);
+
+ matrix = (List<List<Number>>)tuples.get(0).get("i");
+
+ assertEquals(matrix.size(), 3);
+ row = matrix.get(0);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 4.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 8.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 12.0, 0.0);
+
+ row = matrix.get(1);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 5.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 10.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 15.0, 0.0);
+
+ row = matrix.get(2);
+ assertEquals(row.size(), 3);
+ assertEquals(row.get(0).doubleValue(), 6.0, 0.0);
+ assertEquals(row.get(1).doubleValue(), 12.0, 0.0);
+ assertEquals(row.get(2).doubleValue(), 18.0, 0.0);
+ }
+
+ @Test
+ public void testKmeans() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(1,1,1,0,0,0)," +
+ " b=array(1,1,1,0,0,0)," +
+ " c=array(0,0,0,1,1,1)," +
+ " d=array(0,0,0,1,1,1)," +
+ " e=setRowLabels(matrix(a,b,c,d), " +
+ " array(doc1, doc2, doc3, doc4))," +
+ " f=kmeans(e, 2)," +
+ " g=getCluster(f, 0)," +
+ " h=getCluster(f, 1)," +
+ " i=getCentroids(f)," +
+ " j=getRowLabels(g)," +
+ " k=getRowLabels(h))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> cluster1 = (List<List<Number>>)tuples.get(0).get("g");
+ List<List<Number>> cluster2 = (List<List<Number>>)tuples.get(0).get("h");
+ List<List<Number>> centroids = (List<List<Number>>)tuples.get(0).get("i");
+ List<String> labels1 = (List<String>)tuples.get(0).get("j");
+ List<String> labels2 = (List<String>)tuples.get(0).get("k");
+
+ assertEquals(cluster1.size(), 2);
+ assertEquals(cluster2.size(), 2);
+ assertEquals(centroids.size(), 2);
+
+ //Assert that the docs are not in both clusters
+ assertTrue(!(labels1.contains("doc1") && labels2.contains("doc1")));
+ assertTrue(!(labels1.contains("doc2") && labels2.contains("doc2")));
+ assertTrue(!(labels1.contains("doc3") && labels2.contains("doc3")));
+ assertTrue(!(labels1.contains("doc4") && labels2.contains("doc4")));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels1
+ assertTrue((labels1.contains("doc1") && labels1.contains("doc2")) ||
+ ((labels1.contains("doc3") && labels1.contains("doc4"))));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels2
+ assertTrue((labels2.contains("doc1") && labels2.contains("doc2")) ||
+ ((labels2.contains("doc3") && labels2.contains("doc4"))));
+
+ if(labels1.contains("doc1")) {
+ assertEquals(centroids.get(0).get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(1).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(2).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(4).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(5).doubleValue(), 0.0, 0.0);
+
+ assertEquals(centroids.get(1).get(0).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(3).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(4).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(5).doubleValue(), 1.0, 0.0);
+ } else {
+ assertEquals(centroids.get(0).get(0).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(3).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(4).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(5).doubleValue(), 1.0, 0.0);
+
+ assertEquals(centroids.get(1).get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(1).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(2).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(4).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(5).doubleValue(), 0.0, 0.0);
+ }
+ }
+
+ @Test
+ public void testMultiKmeans() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(1,1,1,0,0,0)," +
+ " b=array(1,1,1,0,0,0)," +
+ " c=array(0,0,0,1,1,1)," +
+ " d=array(0,0,0,1,1,1)," +
+ " e=setRowLabels(matrix(a,b,c,d), " +
+ " array(doc1, doc2, doc3, doc4))," +
+ " f=multiKmeans(e, 2, 5)," +
+ " g=getCluster(f, 0)," +
+ " h=getCluster(f, 1)," +
+ " i=getCentroids(f)," +
+ " j=getRowLabels(g)," +
+ " k=getRowLabels(h))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> cluster1 = (List<List<Number>>)tuples.get(0).get("g");
+ List<List<Number>> cluster2 = (List<List<Number>>)tuples.get(0).get("h");
+ List<List<Number>> centroids = (List<List<Number>>)tuples.get(0).get("i");
+ List<String> labels1 = (List<String>)tuples.get(0).get("j");
+ List<String> labels2 = (List<String>)tuples.get(0).get("k");
+
+ assertEquals(cluster1.size(), 2);
+ assertEquals(cluster2.size(), 2);
+ assertEquals(centroids.size(), 2);
+
+ //Assert that the docs are not in both clusters
+ assertTrue(!(labels1.contains("doc1") && labels2.contains("doc1")));
+ assertTrue(!(labels1.contains("doc2") && labels2.contains("doc2")));
+ assertTrue(!(labels1.contains("doc3") && labels2.contains("doc3")));
+ assertTrue(!(labels1.contains("doc4") && labels2.contains("doc4")));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels1
+ assertTrue((labels1.contains("doc1") && labels1.contains("doc2")) ||
+ ((labels1.contains("doc3") && labels1.contains("doc4"))));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels2
+ assertTrue((labels2.contains("doc1") && labels2.contains("doc2")) ||
+ ((labels2.contains("doc3") && labels2.contains("doc4"))));
+
+ if(labels1.contains("doc1")) {
+ assertEquals(centroids.get(0).get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(1).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(2).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(4).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(5).doubleValue(), 0.0, 0.0);
+
+ assertEquals(centroids.get(1).get(0).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(3).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(4).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(5).doubleValue(), 1.0, 0.0);
+ } else {
+ assertEquals(centroids.get(0).get(0).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(1).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(2).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(0).get(3).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(4).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(0).get(5).doubleValue(), 1.0, 0.0);
+
+ assertEquals(centroids.get(1).get(0).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(1).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(2).doubleValue(), 1.0, 0.0);
+ assertEquals(centroids.get(1).get(3).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(4).doubleValue(), 0.0, 0.0);
+ assertEquals(centroids.get(1).get(5).doubleValue(), 0.0, 0.0);
+ }
+ }
+
+ @Test
+ public void testFuzzyKmeans() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=array(1,1,1,0,0,0)," +
+ " b=array(1,1,1,0,0,0)," +
+ " c=array(0,0,0,1,1,1)," +
+ " d=array(0,0,0,1,1,1)," +
+ " e=setRowLabels(matrix(a,b,c,d), " +
+ " array(doc1, doc2, doc3, doc4))," +
+ " f=fuzzyKmeans(e, 2)," +
+ " g=getCluster(f, 0)," +
+ " h=getCluster(f, 1)," +
+ " i=getCentroids(f)," +
+ " j=getRowLabels(g)," +
+ " k=getRowLabels(h)," +
+ " l=getMembershipMatrix(f))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> cluster1 = (List<List<Number>>)tuples.get(0).get("g");
+ List<List<Number>> cluster2 = (List<List<Number>>)tuples.get(0).get("h");
+ List<List<Number>> centroids = (List<List<Number>>)tuples.get(0).get("i");
+ List<List<Number>> membership = (List<List<Number>>)tuples.get(0).get("l");
+
+ List<String> labels1 = (List<String>)tuples.get(0).get("j");
+ List<String> labels2 = (List<String>)tuples.get(0).get("k");
+
+ assertEquals(cluster1.size(), 2);
+ assertEquals(cluster2.size(), 2);
+ assertEquals(centroids.size(), 2);
+
+ //Assert that the docs are not in both clusters
+ assertTrue(!(labels1.contains("doc1") && labels2.contains("doc1")));
+ assertTrue(!(labels1.contains("doc2") && labels2.contains("doc2")));
+ assertTrue(!(labels1.contains("doc3") && labels2.contains("doc3")));
+ assertTrue(!(labels1.contains("doc4") && labels2.contains("doc4")));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels1
+ assertTrue((labels1.contains("doc1") && labels1.contains("doc2")) ||
+ ((labels1.contains("doc3") && labels1.contains("doc4"))));
+
+ //Assert that (doc1 and doc2) or (doc3 and doc4) are in labels2
+ assertTrue((labels2.contains("doc1") && labels2.contains("doc2")) ||
+ ((labels2.contains("doc3") && labels2.contains("doc4"))));
+
+
+ if(labels1.contains("doc1")) {
+ assertEquals(centroids.get(0).get(0).doubleValue(), 1.0, 0.001);
+ assertEquals(centroids.get(0).get(1).doubleValue(), 1.0, 0.001);
+ assertEquals(centroids.get(0).get(2).doubleValue(), 1.0, 0.001);
+ assertEquals(centroids.get(0).get(3).doubleValue(), 0.0, 0.001);
+ assertEquals(centroids.get(0).get(4).doubleValue(), 0.0, 0.001);
+ assertEquals(centroids.get(0).get(5).doubleValue(), 0.0, 0.001);
+
+ assertEquals(centroids.get(1).get(0).doubleValue(), 0.0, 0.001);
+ assertEquals(centroids.get(1).get(1).doubleValue(), 0.0, 0.001);
+ assertEquals(centroids.get(1).get(2).doubleValue(), 0.0,
<TRUNCATED>
[5/8] lucene-solr:master: SOLR-12183: Refactor Streaming Expression
test cases
Posted by jb...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80375acb/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
new file mode 100644
index 0000000..2afc74f
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
@@ -0,0 +1,3954 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.stream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.ComparatorOrder;
+import org.apache.solr.client.solrj.io.comp.FieldComparator;
+import org.apache.solr.client.solrj.io.eval.AddEvaluator;
+import org.apache.solr.client.solrj.io.eval.AndEvaluator;
+import org.apache.solr.client.solrj.io.eval.EqualToEvaluator;
+import org.apache.solr.client.solrj.io.eval.GreaterThanEqualToEvaluator;
+import org.apache.solr.client.solrj.io.eval.GreaterThanEvaluator;
+import org.apache.solr.client.solrj.io.eval.IfThenElseEvaluator;
+import org.apache.solr.client.solrj.io.eval.LessThanEqualToEvaluator;
+import org.apache.solr.client.solrj.io.eval.LessThanEvaluator;
+import org.apache.solr.client.solrj.io.eval.NotEvaluator;
+import org.apache.solr.client.solrj.io.eval.OrEvaluator;
+import org.apache.solr.client.solrj.io.eval.RawValueEvaluator;
+import org.apache.solr.client.solrj.io.ops.ConcatOperation;
+import org.apache.solr.client.solrj.io.ops.GroupOperation;
+import org.apache.solr.client.solrj.io.ops.ReplaceOperation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.stream.metrics.CountMetric;
+import org.apache.solr.client.solrj.io.stream.metrics.MaxMetric;
+import org.apache.solr.client.solrj.io.stream.metrics.MeanMetric;
+import org.apache.solr.client.solrj.io.stream.metrics.MinMetric;
+import org.apache.solr.client.solrj.io.stream.metrics.SumMetric;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@Slow
+@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
+public class StreamDecoratorTest extends SolrCloudTestCase {
+
+ private static final String COLLECTIONORALIAS = "collection1";
+ private static final int TIMEOUT = DEFAULT_TIMEOUT;
+ private static final String id = "id";
+
+ private static boolean useAlias;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(4)
+ .addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
+ .addConfig("ml", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("ml").resolve("conf"))
+ .configure();
+
+ String collection;
+ useAlias = random().nextBoolean();
+ if (useAlias) {
+ collection = COLLECTIONORALIAS + "_collection";
+ } else {
+ collection = COLLECTIONORALIAS;
+ }
+
+ CollectionAdminRequest.createCollection(collection, "conf", 2, 1).process(cluster.getSolrClient());
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(),
+ false, true, TIMEOUT);
+ if (useAlias) {
+ CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection).process(cluster.getSolrClient());
+ }
+ }
+
+ @Before
+ public void cleanIndex() throws Exception {
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+ }
+
+ @Test
+ public void testUniqueStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class);
+
+ try {
+ // Basic test
+ expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f\")");
+ stream = new UniqueStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 0, 1, 3, 4);
+
+ // Basic test desc
+ expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc, a_i desc\"), over=\"a_f\")");
+ stream = new UniqueStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 4, 3, 1, 2);
+
+ // Basic w/multi comp
+ expression = StreamExpressionParser.parse("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f, a_i\")");
+ stream = new UniqueStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 5);
+ assertOrder(tuples, 0, 2, 1, 3, 4);
+
+ // full factory w/multi comp
+ stream = factory.constructStream("unique(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), over=\"a_f, a_i\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 5);
+ assertOrder(tuples, 0, 2, 1, 3, 4);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testSortStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ try {
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("sort", SortStream.class);
+
+ // Basic test
+ stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 6);
+ assertOrder(tuples, 0, 1, 5, 2, 3, 4);
+
+ // Basic test desc
+ stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i desc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 6);
+ assertOrder(tuples, 4, 3, 2, 1, 5, 0);
+
+ // Basic w/multi comp
+ stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc, a_f desc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 6);
+ assertOrder(tuples, 0, 5, 1, 2, 3, 4);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testNullStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("null", NullStream.class);
+
+ try {
+ // Basic test
+ stream = factory.constructStream("null(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assertTrue(tuples.size() == 1);
+ assertTrue(tuples.get(0).getLong("nullCount") == 6);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testParallelNullStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("null", NullStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+ try {
+
+ // Basic test
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"nullCount desc\", null(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), by=\"a_i asc\"))");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assertTrue(tuples.size() == 2);
+ long nullCount = 0;
+ for (Tuple t : tuples) {
+ nullCount += t.getLong("nullCount");
+ }
+
+ assertEquals(nullCount, 6L);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testMergeStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("merge", MergeStream.class);
+
+ // Basic test
+ expression = StreamExpressionParser.parse("merge("
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"),"
+ + "on=\"a_f asc\")");
+
+ stream = new MergeStream(expression, factory);
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ try {
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 0, 1, 3, 4);
+
+ // Basic test desc
+ expression = StreamExpressionParser.parse("merge("
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
+ + "on=\"a_f desc\")");
+ stream = new MergeStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 4, 3, 1, 0);
+
+ // Basic w/multi comp
+ expression = StreamExpressionParser.parse("merge("
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "on=\"a_f asc, a_s asc\")");
+ stream = new MergeStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 5);
+ assertOrder(tuples, 0, 2, 1, 3, 4);
+
+ // full factory w/multi comp
+ stream = factory.constructStream("merge("
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "on=\"a_f asc, a_s asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 5);
+ assertOrder(tuples, 0, 2, 1, 3, 4);
+
+ // full factory w/multi streams
+ stream = factory.constructStream("merge("
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"id:(2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\"),"
+ + "on=\"a_f asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 0, 2, 1, 4);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testRankStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("top", RankStream.class);
+ try {
+ // Basic test
+ expression = StreamExpressionParser.parse("top("
+ + "n=3,"
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"),"
+ + "sort=\"a_f asc, a_i asc\")");
+ stream = new RankStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 3);
+ assertOrder(tuples, 0, 2, 1);
+
+ // Basic test desc
+ expression = StreamExpressionParser.parse("top("
+ + "n=2,"
+ + "unique("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\"),"
+ + "over=\"a_f\"),"
+ + "sort=\"a_f desc\")");
+ stream = new RankStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 2);
+ assertOrder(tuples, 4, 3);
+
+ // full factory
+ stream = factory.constructStream("top("
+ + "n=4,"
+ + "unique("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"),"
+ + "over=\"a_f\"),"
+ + "sort=\"a_f asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 0, 1, 3, 4);
+
+ // full factory, switch order
+ stream = factory.constructStream("top("
+ + "n=4,"
+ + "unique("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc, a_i desc\"),"
+ + "over=\"a_f\"),"
+ + "sort=\"a_f asc\")");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 4);
+ assertOrder(tuples, 2, 1, 3, 4);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testReducerStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ Tuple t0, t1, t2;
+ List<Map> maps0, maps1, maps2;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("reduce", ReducerStream.class)
+ .withFunctionName("group", GroupOperation.class);
+
+ try {
+ // basic
+ expression = StreamExpressionParser.parse("reduce("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc, a_f asc\"),"
+ + "by=\"a_s\","
+ + "group(sort=\"a_f desc\", n=\"4\"))");
+
+ stream = factory.constructStream(expression);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 3);
+
+ t0 = tuples.get(0);
+ maps0 = t0.getMaps("group");
+ assertMaps(maps0, 9, 1, 2, 0);
+
+ t1 = tuples.get(1);
+ maps1 = t1.getMaps("group");
+ assertMaps(maps1, 8, 7, 5, 3);
+
+
+ t2 = tuples.get(2);
+ maps2 = t2.getMaps("group");
+ assertMaps(maps2, 6, 4);
+
+ // basic w/spaces
+ expression = StreamExpressionParser.parse("reduce("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc, a_f asc\"),"
+ + "by=\"a_s\"," +
+ "group(sort=\"a_i asc\", n=\"2\"))");
+ stream = factory.constructStream(expression);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 3);
+
+ t0 = tuples.get(0);
+ maps0 = t0.getMaps("group");
+ assert (maps0.size() == 2);
+
+ assertMaps(maps0, 0, 1);
+
+ t1 = tuples.get(1);
+ maps1 = t1.getMaps("group");
+ assertMaps(maps1, 3, 5);
+
+ t2 = tuples.get(2);
+ maps2 = t2.getMaps("group");
+ assertMaps(maps2, 4, 6);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testHavingStream() throws Exception {
+
+ SolrClientCache solrClientCache = new SolrClientCache();
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
+ .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
+ .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
+ .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
+ .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
+ .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("having", HavingStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("and", AndEvaluator.class)
+ .withFunctionName("or", OrEvaluator.class)
+ .withFunctionName("not", NotEvaluator.class)
+ .withFunctionName("gt", GreaterThanEvaluator.class)
+ .withFunctionName("lt", LessThanEvaluator.class)
+ .withFunctionName("eq", EqualToEvaluator.class)
+ .withFunctionName("lteq", LessThanEqualToEvaluator.class)
+ .withFunctionName("gteq", GreaterThanEqualToEvaluator.class);
+
+ stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), eq(a_i, 9))");
+ StreamContext context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+ Tuple t = tuples.get(0);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(eq(a_i, 9),lt(a_i, 10)))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), or(eq(a_i, 9),eq(a_i, 8)))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 2);
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("8"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("id").equals("9"));
+
+
+ stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(eq(a_i, 9),not(eq(a_i, 9))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 0);
+
+ stream = factory.constructStream("having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), and(lteq(a_i, 9), gteq(a_i, 8)))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 2);
+
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("8"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("having(rollup(over=a_f, sum(a_i), search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\")), and(eq(sum(a_i), 9),eq(sum(a_i), 9)))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+ t = tuples.get(0);
+ assertTrue(t.getDouble("a_f") == 10.0D);
+
+ solrClientCache.close();
+ }
+
+ @Test
+ public void testParallelHavingStream() throws Exception {
+
+ SolrClientCache solrClientCache = new SolrClientCache();
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
+ .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
+ .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
+ .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
+ .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
+ .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("having", HavingStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("and", AndEvaluator.class)
+ .withFunctionName("or", OrEvaluator.class)
+ .withFunctionName("not", NotEvaluator.class)
+ .withFunctionName("gt", GreaterThanEvaluator.class)
+ .withFunctionName("lt", LessThanEvaluator.class)
+ .withFunctionName("eq", EqualToEvaluator.class)
+ .withFunctionName("lteq", LessThanEqualToEvaluator.class)
+ .withFunctionName("gteq", GreaterThanEqualToEvaluator.class)
+ .withFunctionName("val", RawValueEvaluator.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), eq(a_i, 9)))");
+ StreamContext context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+ Tuple t = tuples.get(0);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(eq(a_i, 9),lt(a_i, 10))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\",having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), or(eq(a_i, 9),eq(a_i, 8))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 2);
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("8"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("id").equals("9"));
+
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(eq(a_i, 9),not(eq(a_i, 9)))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 0);
+
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\",having(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=id), and(lteq(a_i, 9), gteq(a_i, 8))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 2);
+
+ t = tuples.get(0);
+ assertTrue(t.getString("id").equals("8"));
+
+ t = tuples.get(1);
+ assertTrue(t.getString("id").equals("9"));
+
+ stream = factory.constructStream("parallel("+COLLECTIONORALIAS+", workers=2, sort=\"a_f asc\", having(rollup(over=a_f, sum(a_i), search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=a_f)), and(eq(sum(a_i), 9),eq(sum(a_i),9))))");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 1);
+
+ t = tuples.get(0);
+ assertTrue(t.getDouble("a_f") == 10.0D);
+
+ solrClientCache.close();
+ }
+
+ @Test
+ public void testFetchStream() throws Exception {
+
+ SolrClientCache solrClientCache = new SolrClientCache();//TODO share in @Before ; close in @After ?
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
+ .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
+ .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
+ .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
+ .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
+ .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("fetch", FetchStream.class);
+
+ stream = factory.constructStream("fetch("+ COLLECTIONORALIAS +", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), on=\"id=a_i\", batchSize=\"2\", fl=\"subject\")");
+ StreamContext context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 10);
+ Tuple t = tuples.get(0);
+ assertTrue("blah blah blah 0".equals(t.getString("subject")));
+ t = tuples.get(1);
+ assertTrue("blah blah blah 2".equals(t.getString("subject")));
+ t = tuples.get(2);
+ assertTrue("blah blah blah 3".equals(t.getString("subject")));
+ t = tuples.get(3);
+ assertTrue("blah blah blah 4".equals(t.getString("subject")));
+ t = tuples.get(4);
+ assertTrue("blah blah blah 1".equals(t.getString("subject")));
+ t = tuples.get(5);
+ assertTrue("blah blah blah 5".equals(t.getString("subject")));
+ t = tuples.get(6);
+ assertTrue("blah blah blah 6".equals(t.getString("subject")));
+ t = tuples.get(7);
+ assertTrue("blah blah blah 7".equals(t.getString("subject")));
+ t = tuples.get(8);
+ assertTrue("blah blah blah 8".equals(t.getString("subject")));
+ t = tuples.get(9);
+ assertTrue("blah blah blah 9".equals(t.getString("subject")));
+
+ //Change the batch size
+ stream = factory.constructStream("fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), on=\"id=a_i\", batchSize=\"3\", fl=\"subject\")");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 10);
+ t = tuples.get(0);
+ assertTrue("blah blah blah 0".equals(t.getString("subject")));
+ t = tuples.get(1);
+ assertTrue("blah blah blah 2".equals(t.getString("subject")));
+ t = tuples.get(2);
+ assertTrue("blah blah blah 3".equals(t.getString("subject")));
+ t = tuples.get(3);
+ assertTrue("blah blah blah 4".equals(t.getString("subject")));
+ t = tuples.get(4);
+ assertTrue("blah blah blah 1".equals(t.getString("subject")));
+ t = tuples.get(5);
+ assertTrue("blah blah blah 5".equals(t.getString("subject")));
+ t = tuples.get(6);
+ assertTrue("blah blah blah 6".equals(t.getString("subject")));
+ t = tuples.get(7);
+ assertTrue("blah blah blah 7".equals(t.getString("subject")));
+ t = tuples.get(8);
+ assertTrue("blah blah blah 8".equals(t.getString("subject")));
+ t = tuples.get(9);
+ assertTrue("blah blah blah 9".equals(t.getString("subject")));
+
+ // SOLR-10404 test that "hello 99" as a value gets escaped
+ new UpdateRequest()
+ .add(id, "99", "a1_s", "hello 99", "a2_s", "hello 99", "subject", "blah blah blah 99")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ stream = factory.constructStream("fetch("+ COLLECTIONORALIAS +", search(" + COLLECTIONORALIAS + ", q=" + id + ":99, fl=\"id,a1_s\", sort=\"id asc\"), on=\"a1_s=a2_s\", fl=\"subject\")");
+ context = new StreamContext();
+ context.setSolrClientCache(solrClientCache);
+ stream.setStreamContext(context);
+ tuples = getTuples(stream);
+
+ assertEquals(1, tuples.size());
+ t = tuples.get(0);
+ assertTrue("blah blah blah 99".equals(t.getString("subject")));
+
+ solrClientCache.close();
+ }
+
+ @Test
+ public void testParallelFetchStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "subject", "blah blah blah 0")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "subject", "blah blah blah 2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "subject", "blah blah blah 3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "subject", "blah blah blah 4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "subject", "blah blah blah 1")
+ .add(id, "5", "a_s", "hello3", "a_i", "5", "a_f", "6", "subject", "blah blah blah 5")
+ .add(id, "6", "a_s", "hello4", "a_i", "6", "a_f", "7", "subject", "blah blah blah 6")
+ .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "8", "subject", "blah blah blah 7")
+ .add(id, "8", "a_s", "hello3", "a_i", "8", "a_f", "9", "subject", "blah blah blah 8")
+ .add(id, "9", "a_s", "hello0", "a_i", "9", "a_f", "10", "subject", "blah blah blah 9")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("parallel", ParallelStream.class)
+ .withFunctionName("fetch", FetchStream.class);
+
+ try {
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"2\", fl=\"subject\"))");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 10);
+ Tuple t = tuples.get(0);
+ assertTrue("blah blah blah 0".equals(t.getString("subject")));
+ t = tuples.get(1);
+ assertTrue("blah blah blah 2".equals(t.getString("subject")));
+ t = tuples.get(2);
+ assertTrue("blah blah blah 3".equals(t.getString("subject")));
+ t = tuples.get(3);
+ assertTrue("blah blah blah 4".equals(t.getString("subject")));
+ t = tuples.get(4);
+ assertTrue("blah blah blah 1".equals(t.getString("subject")));
+ t = tuples.get(5);
+ assertTrue("blah blah blah 5".equals(t.getString("subject")));
+ t = tuples.get(6);
+ assertTrue("blah blah blah 6".equals(t.getString("subject")));
+ t = tuples.get(7);
+ assertTrue("blah blah blah 7".equals(t.getString("subject")));
+ t = tuples.get(8);
+ assertTrue("blah blah blah 8".equals(t.getString("subject")));
+ t = tuples.get(9);
+ assertTrue("blah blah blah 9".equals(t.getString("subject")));
+
+
+ stream = factory.constructStream("parallel(" + COLLECTIONORALIAS + ", workers=2, sort=\"a_f asc\", fetch(" + COLLECTIONORALIAS + ", search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\", partitionKeys=\"id\"), on=\"id=a_i\", batchSize=\"3\", fl=\"subject\"))");
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 10);
+ t = tuples.get(0);
+ assertTrue("blah blah blah 0".equals(t.getString("subject")));
+ t = tuples.get(1);
+ assertTrue("blah blah blah 2".equals(t.getString("subject")));
+ t = tuples.get(2);
+ assertTrue("blah blah blah 3".equals(t.getString("subject")));
+ t = tuples.get(3);
+ assertTrue("blah blah blah 4".equals(t.getString("subject")));
+ t = tuples.get(4);
+ assertTrue("blah blah blah 1".equals(t.getString("subject")));
+ t = tuples.get(5);
+ assertTrue("blah blah blah 5".equals(t.getString("subject")));
+ t = tuples.get(6);
+ assertTrue("blah blah blah 6".equals(t.getString("subject")));
+ t = tuples.get(7);
+ assertTrue("blah blah blah 7".equals(t.getString("subject")));
+ t = tuples.get(8);
+ assertTrue("blah blah blah 8".equals(t.getString("subject")));
+ t = tuples.get(9);
+ assertTrue("blah blah blah 9".equals(t.getString("subject")));
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testDaemonStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class)
+ .withFunctionName("daemon", DaemonStream.class);
+
+ StreamExpression expression;
+ DaemonStream daemonStream;
+
+ expression = StreamExpressionParser.parse("daemon(rollup("
+ + "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"a_i,a_s\", sort=\"a_s asc\"),"
+ + "over=\"a_s\","
+ + "sum(a_i)"
+ + "), id=\"test\", runInterval=\"1000\", queueSize=\"9\")");
+ daemonStream = (DaemonStream)factory.constructStream(expression);
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ daemonStream.setStreamContext(streamContext);
+ try {
+ //Test Long and Double Sums
+
+ daemonStream.open(); // This will start the daemon thread
+
+ for (int i = 0; i < 4; i++) {
+ Tuple tuple = daemonStream.read(); // Reads from the queue
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 1:"+bucket);
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 2:"+bucket);
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ //System.out.println("#################################### Bucket 3:"+bucket);
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ }
+
+ //Now lets wait until the internal queue fills up
+
+ while (daemonStream.remainingCapacity() > 0) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+
+ }
+ }
+
+ //OK capacity is full, let's index a new doc
+
+ new UpdateRequest()
+ .add(id, "10", "a_s", "hello0", "a_i", "1", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ //Now lets clear the existing docs in the queue 9, plus 3 more to get passed the run that was blocked. The next run should
+ //have the tuples with the updated count.
+ for (int i = 0; i < 12; i++) {
+ daemonStream.read();
+ }
+
+ //And rerun the loop. It should have a new count for hello0
+ for (int i = 0; i < 4; i++) {
+ Tuple tuple = daemonStream.read(); // Reads from the queue
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 18.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ }
+ } finally {
+ daemonStream.close(); //This should stop the daemon thread
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testTerminatingDaemonStream() throws Exception {
+ Assume.assumeTrue(!useAlias);
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("topic", TopicStream.class)
+ .withFunctionName("daemon", DaemonStream.class);
+
+ StreamExpression expression;
+ DaemonStream daemonStream;
+
+ SolrClientCache cache = new SolrClientCache();
+ StreamContext context = new StreamContext();
+ context.setSolrClientCache(cache);
+ expression = StreamExpressionParser.parse("daemon(topic("+ COLLECTIONORALIAS +","+ COLLECTIONORALIAS +", q=\"a_s:hello\", initialCheckpoint=0, id=\"topic1\", rows=2, fl=\"id\""
+ + "), id=test, runInterval=1000, terminate=true, queueSize=50)");
+ daemonStream = (DaemonStream)factory.constructStream(expression);
+ daemonStream.setStreamContext(context);
+
+ List<Tuple> tuples = getTuples(daemonStream);
+ assertTrue(tuples.size() == 10);
+ cache.close();
+ }
+
+ @Test
+ public void testRollupStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ try {
+ expression = StreamExpressionParser.parse("rollup("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s asc\"),"
+ + "over=\"a_s\","
+ + "sum(a_i),"
+ + "sum(a_f),"
+ + "min(a_i),"
+ + "min(a_f),"
+ + "max(a_i),"
+ + "max(a_f),"
+ + "avg(a_i),"
+ + "avg(a_f),"
+ + "count(*),"
+ + ")");
+ stream = factory.constructStream(expression);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 3);
+
+ //Test Long and Double Sums
+
+ Tuple tuple = tuples.get(0);
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+ Double sumf = tuple.getDouble("sum(a_f)");
+ Double mini = tuple.getDouble("min(a_i)");
+ Double minf = tuple.getDouble("min(a_f)");
+ Double maxi = tuple.getDouble("max(a_i)");
+ Double maxf = tuple.getDouble("max(a_f)");
+ Double avgi = tuple.getDouble("avg(a_i)");
+ Double avgf = tuple.getDouble("avg(a_f)");
+ Double count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
+
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
+
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
+
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testParallelUniqueStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1")
+ .add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5")
+ .add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5")
+ .add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String zkHost = cluster.getZkServer().getZkAddress();
+ StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("top", RankStream.class)
+ .withFunctionName("group", ReducerStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+
+
+ try {
+
+ ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", unique(search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\"), over=\"a_f\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_f asc\")");
+ pstream.setStreamContext(streamContext);
+ List<Tuple> tuples = getTuples(pstream);
+ assert (tuples.size() == 5);
+ assertOrder(tuples, 0, 1, 3, 4, 6);
+
+ //Test the eofTuples
+
+ Map<String, Tuple> eofTuples = pstream.getEofTuples();
+ assert (eofTuples.size() == 2); //There should be an EOF tuple for each worker.
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testParallelShuffleStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1")
+ .add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5")
+ .add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5")
+ .add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "9", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "10", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "11", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "12", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "13", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "14", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "15", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "16", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "17", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "18", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "19", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "20", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "21", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "22", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "23", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "24", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "25", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "26", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "27", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "28", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "29", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "30", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "31", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "32", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "33", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "34", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "35", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "36", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "37", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "38", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "39", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "40", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "41", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "42", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "43", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "44", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "45", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "46", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "47", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "48", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "49", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "50", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "51", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "52", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "53", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "54", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "55", "a_s", "hello1", "a_i", "13", "a_f", "4")
+ .add(id, "56", "a_s", "hello1", "a_i", "13", "a_f", "1000")
+
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ String zkHost = cluster.getZkServer().getZkAddress();
+ StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
+ .withFunctionName("shuffle", ShuffleStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+ try {
+ ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", unique(shuffle(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\"), over=\"a_f\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_f asc\")");
+ pstream.setStreamFactory(streamFactory);
+ pstream.setStreamContext(streamContext);
+ List<Tuple> tuples = getTuples(pstream);
+ assert (tuples.size() == 6);
+ assertOrder(tuples, 0, 1, 3, 4, 6, 56);
+
+ //Test the eofTuples
+
+ Map<String, Tuple> eofTuples = pstream.getEofTuples();
+ assert (eofTuples.size() == 2); //There should be an EOF tuple for each worker.
+ assert (pstream.toExpression(streamFactory).toString().contains("shuffle"));
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testParallelReducerStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+
+ String zkHost = cluster.getZkServer().getZkAddress();
+ StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("group", GroupOperation.class)
+ .withFunctionName("reduce", ReducerStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+
+ try {
+ ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", " +
+ "reduce(" +
+ "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"id,a_s,a_i,a_f\", sort=\"a_s asc,a_f asc\", partitionKeys=\"a_s\"), " +
+ "by=\"a_s\"," +
+ "group(sort=\"a_i asc\", n=\"5\")), " +
+ "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_s asc\")");
+
+ pstream.setStreamContext(streamContext);
+
+ List<Tuple> tuples = getTuples(pstream);
+
+ assert (tuples.size() == 3);
+
+ Tuple t0 = tuples.get(0);
+ List<Map> maps0 = t0.getMaps("group");
+ assertMaps(maps0, 0, 1, 2, 9);
+
+ Tuple t1 = tuples.get(1);
+ List<Map> maps1 = t1.getMaps("group");
+ assertMaps(maps1, 3, 5, 7, 8);
+
+ Tuple t2 = tuples.get(2);
+ List<Map> maps2 = t2.getMaps("group");
+ assertMaps(maps2, 4, 6);
+
+
+ pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", " +
+ "reduce(" +
+ "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"id,a_s,a_i,a_f\", sort=\"a_s desc,a_f asc\", partitionKeys=\"a_s\"), " +
+ "by=\"a_s\", " +
+ "group(sort=\"a_i desc\", n=\"5\"))," +
+ "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_s desc\")");
+
+ pstream.setStreamContext(streamContext);
+ tuples = getTuples(pstream);
+
+ assert (tuples.size() == 3);
+
+ t0 = tuples.get(0);
+ maps0 = t0.getMaps("group");
+ assertMaps(maps0, 6, 4);
+
+
+ t1 = tuples.get(1);
+ maps1 = t1.getMaps("group");
+ assertMaps(maps1, 8, 7, 5, 3);
+
+
+ t2 = tuples.get(2);
+ maps2 = t2.getMaps("group");
+ assertMaps(maps2, 9, 2, 1, 0);
+ } finally {
+ solrClientCache.close();
+ }
+
+ }
+
+ @Test
+ public void testParallelRankStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "5", "a_s", "hello1", "a_i", "5", "a_f", "1")
+ .add(id, "6", "a_s", "hello1", "a_i", "6", "a_f", "1")
+ .add(id, "7", "a_s", "hello1", "a_i", "7", "a_f", "1")
+ .add(id, "8", "a_s", "hello1", "a_i", "8", "a_f", "1")
+ .add(id, "9", "a_s", "hello1", "a_i", "9", "a_f", "1")
+ .add(id, "10", "a_s", "hello1", "a_i", "10", "a_f", "1")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String zkHost = cluster.getZkServer().getZkAddress();
+ StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("top", RankStream.class)
+ .withFunctionName("group", ReducerStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ try {
+ ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel("
+ + COLLECTIONORALIAS + ", "
+ + "top("
+ + "search(" + COLLECTIONORALIAS + ", q=\"*:*\", fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\"), "
+ + "n=\"11\", "
+ + "sort=\"a_i desc\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i desc\")");
+ pstream.setStreamContext(streamContext);
+ List<Tuple> tuples = getTuples(pstream);
+
+ assert (tuples.size() == 10);
+ assertOrder(tuples, 10, 9, 8, 7, 6, 5, 4, 3, 2, 0);
+ } finally {
+ solrClientCache.close();
+ }
+
+ }
+
+ @Test
+ public void testParallelMergeStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
+ .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
+ .add(id, "5", "a_s", "hello0", "a_i", "10", "a_f", "0")
+ .add(id, "6", "a_s", "hello2", "a_i", "8", "a_f", "0")
+ .add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "3")
+ .add(id, "8", "a_s", "hello4", "a_i", "11", "a_f", "4")
+ .add(id, "9", "a_s", "hello1", "a_i", "100", "a_f", "1")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String zkHost = cluster.getZkServer().getZkAddress();
+ StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost)
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("unique", UniqueStream.class)
+ .withFunctionName("top", RankStream.class)
+ .withFunctionName("group", ReducerStream.class)
+ .withFunctionName("merge", MergeStream.class)
+ .withFunctionName("parallel", ParallelStream.class);
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+ try {
+ //Test ascending
+ ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", merge(search(" + COLLECTIONORALIAS + ", q=\"id:(4 1 8 7 9)\", fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\"), search(" + COLLECTIONORALIAS + ", q=\"id:(0 2 3 6)\", fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\"), on=\"a_i asc\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i asc\")");
+ pstream.setStreamContext(streamContext);
+ List<Tuple> tuples = getTuples(pstream);
+
+ assert (tuples.size() == 9);
+ assertOrder(tuples, 0, 1, 2, 3, 4, 7, 6, 8, 9);
+
+ //Test descending
+
+ pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", merge(search(" + COLLECTIONORALIAS + ", q=\"id:(4 1 8 9)\", fl=\"id,a_s,a_i\", sort=\"a_i desc\", partitionKeys=\"a_i\"), search(" + COLLECTIONORALIAS + ", q=\"id:(0 2 3 6)\", fl=\"id,a_s,a_i\", sort=\"a_i desc\", partitionKeys=\"a_i\"), on=\"a_i desc\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i desc\")");
+ pstream.setStreamContext(streamContext);
+ tuples = getTuples(pstream);
+
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 9, 8, 6, 4, 3, 2, 1, 0);
+ } finally {
+ solrClientCache.close();
+ }
+
+ }
+
+ @Test
+ public void testParallelRollupStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("parallel", ParallelStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class);
+
+
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+
+ try {
+ expression = StreamExpressionParser.parse("parallel(" + COLLECTIONORALIAS + ","
+ + "rollup("
+ + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s asc\", partitionKeys=\"a_s\"),"
+ + "over=\"a_s\","
+ + "sum(a_i),"
+ + "sum(a_f),"
+ + "min(a_i),"
+ + "min(a_f),"
+ + "max(a_i),"
+ + "max(a_f),"
+ + "avg(a_i),"
+ + "avg(a_f),"
+ + "count(*)"
+ + "),"
+ + "workers=\"2\", zkHost=\"" + cluster.getZkServer().getZkAddress() + "\", sort=\"a_s asc\")"
+ );
+
+
+ stream = factory.constructStream(expression);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 3);
+
+ //Test Long and Double Sums
+
+ Tuple tuple = tuples.get(0);
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+ Double sumf = tuple.getDouble("sum(a_f)");
+ Double mini = tuple.getDouble("min(a_i)");
+ Double minf = tuple.getDouble("min(a_f)");
+ Double maxi = tuple.getDouble("max(a_i)");
+ Double maxf = tuple.getDouble("max(a_f)");
+ Double avgi = tuple.getDouble("avg(a_i)");
+ Double avgf = tuple.getDouble("avg(a_f)");
+ Double count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+ assertTrue(sumf.doubleValue() == 18.0D);
+ assertTrue(mini.doubleValue() == 0.0D);
+ assertTrue(minf.doubleValue() == 1.0D);
+ assertTrue(maxi.doubleValue() == 14.0D);
+ assertTrue(maxf.doubleValue() == 10.0D);
+ assertTrue(avgi.doubleValue() == 4.25D);
+ assertTrue(avgf.doubleValue() == 4.5D);
+ assertTrue(count.doubleValue() == 4);
+
+ tuple = tuples.get(1);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+ assertTrue(sumf.doubleValue() == 26.0D);
+ assertTrue(mini.doubleValue() == 3.0D);
+ assertTrue(minf.doubleValue() == 3.0D);
+ assertTrue(maxi.doubleValue() == 13.0D);
+ assertTrue(maxf.doubleValue() == 9.0D);
+ assertTrue(avgi.doubleValue() == 9.5D);
+ assertTrue(avgf.doubleValue() == 6.5D);
+ assertTrue(count.doubleValue() == 4);
+
+ tuple = tuples.get(2);
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ sumf = tuple.getDouble("sum(a_f)");
+ mini = tuple.getDouble("min(a_i)");
+ minf = tuple.getDouble("min(a_f)");
+ maxi = tuple.getDouble("max(a_i)");
+ maxf = tuple.getDouble("max(a_f)");
+ avgi = tuple.getDouble("avg(a_i)");
+ avgf = tuple.getDouble("avg(a_f)");
+ count = tuple.getDouble("count(*)");
+
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ assertTrue(sumf.doubleValue() == 11.0D);
+ assertTrue(mini.doubleValue() == 4.0D);
+ assertTrue(minf.doubleValue() == 4.0D);
+ assertTrue(maxi.doubleValue() == 11.0D);
+ assertTrue(maxf.doubleValue() == 7.0D);
+ assertTrue(avgi.doubleValue() == 7.5D);
+ assertTrue(avgf.doubleValue() == 5.5D);
+ assertTrue(count.doubleValue() == 2);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testInnerJoinStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2")
+ .add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", "left_3") // 10
+ .add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", "left_4") // 11
+ .add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", "left_5") // 12
+ .add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6")
+ .add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", "left_7") // 14
+
+ .add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", "0") // 1,15
+ .add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", "0") // 1,15
+ .add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", "1") // 3
+ .add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", "1") // 4
+ .add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", "1") // 5
+ .add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2")
+ .add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", "3") // 7
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("innerJoin", InnerJoinStream.class);
+
+ try {
+ // Basic test
+ expression = StreamExpressionParser.parse("innerJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\"),"
+ + "on=\"join1_i=join1_i, join2_s=join2_s\")");
+ stream = new InnerJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 1, 1, 15, 15, 3, 4, 5, 7);
+
+ // Basic desc
+ expression = StreamExpressionParser.parse("innerJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "on=\"join1_i=join1_i, join2_s=join2_s\")");
+ stream = new InnerJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 7, 3, 4, 5, 1, 1, 15, 15);
+
+ // Results in both searches, no join matches
+ expression = StreamExpressionParser.parse("innerJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\", aliases=\"id=right.id, join1_i=right.join1_i, join2_s=right.join2_s, ident_s=right.ident_s\"),"
+ + "on=\"ident_s=right.ident_s\")");
+ stream = new InnerJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 0);
+
+ // Differing field names
+ expression = StreamExpressionParser.parse("innerJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join3_i asc, join2_s asc\", aliases=\"join3_i=aliasesField\"),"
+ + "on=\"join1_i=aliasesField, join2_s=join2_s\")");
+ stream = new InnerJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 1, 1, 15, 15, 3, 4, 5, 7);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testLeftOuterJoinStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2")
+ .add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", "left_3") // 10
+ .add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", "left_4") // 11
+ .add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", "left_5") // 12
+ .add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6")
+ .add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", "left_7") // 14
+
+ .add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", "0") // 1,15
+ .add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", "0") // 1,15
+ .add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", "1") // 3
+ .add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", "1") // 4
+ .add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", "1") // 5
+ .add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2")
+ .add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", "3") // 7
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("leftOuterJoin", LeftOuterJoinStream.class);
+
+ // Basic test
+ try {
+ expression = StreamExpressionParser.parse("leftOuterJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\"),"
+ + "on=\"join1_i=join1_i, join2_s=join2_s\")");
+ stream = new LeftOuterJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 10);
+ assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
+
+ // Basic desc
+ expression = StreamExpressionParser.parse("leftOuterJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "on=\"join1_i=join1_i, join2_s=join2_s\")");
+ stream = new LeftOuterJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 10);
+ assertOrder(tuples, 7, 6, 3, 4, 5, 1, 1, 15, 15, 2);
+
+ // Results in both searches, no join matches
+ expression = StreamExpressionParser.parse("leftOuterJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\", aliases=\"id=right.id, join1_i=right.join1_i, join2_s=right.join2_s, ident_s=right.ident_s\"),"
+ + "on=\"ident_s=right.ident_s\")");
+ stream = new LeftOuterJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 1, 15, 2, 3, 4, 5, 6, 7);
+
+ // Differing field names
+ expression = StreamExpressionParser.parse("leftOuterJoin("
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\"),"
+ + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join3_i asc, join2_s asc\", aliases=\"join3_i=aliasesField\"),"
+ + "on=\"join1_i=aliasesField, join2_s=join2_s\")");
+ stream = new LeftOuterJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 10);
+ assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
+ } finally {
+ solrClientCache.close();
+ }
+ }
+
+ @Test
+ public void testHashJoinStream() throws Exception {
+
+ new UpdateRequest()
+ .add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", "left_1") // 8, 9
+ .add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2")
+ .add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", "left_3") // 10
+ .add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", "left_4") // 11
+ .add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", "left_5") // 12
+ .add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6")
+ .add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", "left_7") // 14
+
+ .add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", "0") // 1,15
+ .add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", "0") // 1,15
+ .add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", "1") // 3
+ .add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", "1") // 4
+ .add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", "1") // 5
+ .add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2")
+ .add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", "3") // 7
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ StreamContext streamContext = new StreamContext();
+ SolrClientCache solrClientCache = new SolrClientCache();
+ streamContext.setSolrClientCache(solrClientCache);
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("hashJoin", HashJoinStream.class);
+ try {
+ // Basic test
+ expression = StreamExpressionParser.parse("hashJoin("
+ + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\"),"
+ + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\"),"
+ + "on=\"join1_i, join2_s\")");
+ stream = new HashJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples(stream);
+ assert (tuples.size() == 8);
+ assertOrder(tuples, 1, 1, 15, 15, 3, 4, 5, 7);
+
+ // Basic desc
+ expression = StreamExpressionParser.parse("hashJoin("
+ + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\"),"
+ + "on=\"join1_i, join2_s\")");
+ stream = new HashJoinStream(expression, factory);
+ stream.setStreamContext(streamContext);
+ tuples = getTuples
<TRUNCATED>