You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2016/01/22 20:07:16 UTC
svn commit: r1726291 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/handler/
solrj/src/java/org/apache/solr/client/solrj/io/stream/
solrj/src/test/org/apache/solr/client/solrj/io/stream/
Author: jbernste
Date: Fri Jan 22 19:07:15 2016
New Revision: 1726291
URL: http://svn.apache.org/viewvc?rev=1726291&view=rev
Log:
SOLR-8550: Add asynchronous DaemonStreams to the Streaming API
Added:
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RollupStream.java
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1726291&r1=1726290&r2=1726291&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Jan 22 19:07:15 2016
@@ -140,6 +140,8 @@ New Features
* SOLR-8556: Add ConcatOperation to be used with the SelectStream (Joel Bernstein, Dennis Gove)
+* SOLR-8550: Add asynchronous DaemonStreams to the Streaming API (Joel Bernstein)
+
Bug Fixes
----------------------
* SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StreamHandler.java?rev=1726291&r1=1726290&r2=1726291&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StreamHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StreamHandler.java Fri Jan 22 19:07:15 2016
@@ -19,7 +19,9 @@ package org.apache.solr.handler;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -31,26 +33,7 @@ import org.apache.solr.client.solrj.io.o
import org.apache.solr.client.solrj.io.ops.DistinctOperation;
import org.apache.solr.client.solrj.io.ops.GroupOperation;
import org.apache.solr.client.solrj.io.ops.ReplaceOperation;
-import org.apache.solr.client.solrj.io.stream.CloudSolrStream;
-import org.apache.solr.client.solrj.io.stream.ComplementStream;
-import org.apache.solr.client.solrj.io.stream.ExceptionStream;
-import org.apache.solr.client.solrj.io.stream.FacetStream;
-import org.apache.solr.client.solrj.io.stream.HashJoinStream;
-import org.apache.solr.client.solrj.io.stream.InnerJoinStream;
-import org.apache.solr.client.solrj.io.stream.IntersectStream;
-import org.apache.solr.client.solrj.io.stream.JDBCStream;
-import org.apache.solr.client.solrj.io.stream.LeftOuterJoinStream;
-import org.apache.solr.client.solrj.io.stream.MergeStream;
-import org.apache.solr.client.solrj.io.stream.OuterHashJoinStream;
-import org.apache.solr.client.solrj.io.stream.ParallelStream;
-import org.apache.solr.client.solrj.io.stream.RankStream;
-import org.apache.solr.client.solrj.io.stream.ReducerStream;
-import org.apache.solr.client.solrj.io.stream.RollupStream;
-import org.apache.solr.client.solrj.io.stream.StatsStream;
-import org.apache.solr.client.solrj.io.stream.StreamContext;
-import org.apache.solr.client.solrj.io.stream.TupleStream;
-import org.apache.solr.client.solrj.io.stream.UniqueStream;
-import org.apache.solr.client.solrj.io.stream.UpdateStream;
+import org.apache.solr.client.solrj.io.stream.*;
import org.apache.solr.client.solrj.io.stream.expr.Expressible;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.apache.solr.client.solrj.io.stream.metrics.CountMetric;
@@ -78,6 +61,7 @@ public class StreamHandler extends Reque
private StreamFactory streamFactory = new StreamFactory();
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private String coreName;
+ private Map<String, DaemonStream> daemons = new HashMap();
public void inform(SolrCore core) {
@@ -123,6 +107,7 @@ public class StreamHandler extends Reque
.withFunctionName("jdbc", JDBCStream.class)
.withFunctionName("intersect", IntersectStream.class)
.withFunctionName("complement", ComplementStream.class)
+ .withFunctionName("daemon", DaemonStream.class)
// metrics
.withFunctionName("min", MinMetric.class)
@@ -167,6 +152,12 @@ public class StreamHandler extends Reque
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
+
+ if(params.get("action") != null) {
+ handleAdmin(req, rsp, params);
+ return;
+ }
+
TupleStream tupleStream = null;
try {
@@ -187,7 +178,46 @@ public class StreamHandler extends Reque
context.setSolrClientCache(clientCache);
context.put("core", this.coreName);
tupleStream.setStreamContext(context);
- rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
+ if(tupleStream instanceof DaemonStream) {
+ DaemonStream daemonStream = (DaemonStream)tupleStream;
+ if(daemons.containsKey(daemonStream.getId())) {
+ daemons.remove(daemonStream.getId()).close();
+ }
+ daemonStream.open(); //This will start the deamonStream
+ daemons.put(daemonStream.getId(), daemonStream);
+ rsp.add("result-set", new DaemonResponseStream("Deamon:"+daemonStream.getId()+" started on "+coreName));
+ } else {
+ rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
+ }
+ }
+
+ private void handleAdmin(SolrQueryRequest req, SolrQueryResponse rsp, SolrParams params) {
+ String action = params.get("action");
+ if("stop".equalsIgnoreCase(action)) {
+ String id = params.get("id");
+ DaemonStream d = daemons.get(id);
+ if(d != null) {
+ d.close();
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " stopped on " + coreName));
+ } else {
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " not found on " + coreName));
+ }
+ } else if("start".equalsIgnoreCase(action)) {
+ String id = params.get("id");
+ DaemonStream d = daemons.get(id);
+ d.open();
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " started on " + coreName));
+ } else if("list".equalsIgnoreCase(action)) {
+ Collection<DaemonStream> vals = daemons.values();
+ rsp.add("result-set", new DaemonCollectionStream(vals));
+ } else if("kill".equalsIgnoreCase(action)) {
+ String id = params.get("id");
+ DaemonStream d = daemons.remove(id);
+ if (d != null) {
+ d.close();
+ }
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " killed on " + coreName));
+ }
}
private SolrParams adjustParams(SolrParams params) {
@@ -238,6 +268,78 @@ public class StreamHandler extends Reque
}
}
+ public static class DaemonCollectionStream extends TupleStream {
+ private Iterator<DaemonStream> it;
+
+ public DaemonCollectionStream(Collection<DaemonStream> col) {
+ this.it = col.iterator();
+ }
+ public StreamComparator getStreamSort() {
+ return null;
+ }
+
+ public void close() {
+ }
+
+ public void open() {
+ }
+
+ public void setStreamContext(StreamContext context) {
+ }
+
+ public List<TupleStream> children() {
+ return null;
+ }
+
+ public Tuple read() {
+ if(it.hasNext()) {
+ return it.next().getInfo();
+ } else {
+ Map m = new HashMap();
+ m.put("EOF", true);
+ return new Tuple(m);
+ }
+ }
+ }
+
+ public static class DaemonResponseStream extends TupleStream {
+ private String message;
+ private boolean sendEOF = false;
+
+ public DaemonResponseStream(String message) {
+ this.message = message;
+ }
+ public StreamComparator getStreamSort() {
+ return null;
+ }
+
+ public void close() {
+ }
+
+ public void open() {
+ }
+
+ public void setStreamContext(StreamContext context) {
+ }
+
+ public List<TupleStream> children() {
+ return null;
+ }
+
+ public Tuple read() {
+ if (sendEOF) {
+ Map m = new HashMap();
+ m.put("EOF", true);
+ return new Tuple(m);
+ } else {
+ sendEOF = true;
+ Map m = new HashMap();
+ m.put("DaemonOp",message);
+ return new Tuple(m);
+ }
+ }
+ }
+
public static class TimerStream extends TupleStream {
private long begin;
Added: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java?rev=1726291&view=auto
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java (added)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java Fri Jan 22 19:07:15 2016
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.solr.client.solrj.io.stream;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.lang.invoke.MethodHandles;
+import java.util.Locale;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Date;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class DaemonStream extends TupleStream implements Expressible {
+
+ private TupleStream tupleStream;
+ private StreamRunner streamRunner;
+ private ArrayBlockingQueue<Tuple> queue;
+ private int queueSize;
+ private boolean eatTuples;
+ private long iterations;
+ private long startTime;
+ private long stopTime;
+ private Exception exception;
+ private long runInterval;
+ private String id;
+ private boolean closed = false;
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public DaemonStream(StreamExpression expression, StreamFactory factory) throws IOException{
+
+ List<StreamExpression> streamExpressions = factory.getExpressionOperandsRepresentingTypes(expression, Expressible.class, TupleStream.class);
+
+ TupleStream tupleStream = factory.constructStream(streamExpressions.get(0));
+
+ StreamExpressionNamedParameter idExpression = factory.getNamedOperand(expression, "id");
+ StreamExpressionNamedParameter runExpression = factory.getNamedOperand(expression, "runInterval");
+ StreamExpressionNamedParameter queueExpression = factory.getNamedOperand(expression, "queueSize");
+
+ String id = null;
+ long runInterval = 0L;
+ int queueSize = 0;
+
+ if(idExpression == null) {
+ throw new IOException("Invalid expression id parameter expected");
+ } else {
+ id = ((StreamExpressionValue) idExpression.getParameter()).getValue();
+ }
+
+ if(runExpression == null) {
+ throw new IOException("Invalid expression runInterval parameter expected");
+ } else {
+ runInterval = Long.parseLong(((StreamExpressionValue) runExpression.getParameter()).getValue());
+ }
+
+ if(queueExpression != null) {
+ queueSize= Integer.parseInt(((StreamExpressionValue)queueExpression.getParameter()).getValue());
+ }
+
+ // validate expression contains only what we want.
+ if(expression.getParameters().size() != streamExpressions.size() + 2 &&
+ expression.getParameters().size() != streamExpressions.size() + 3) {
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - unknown operands found", expression));
+ }
+
+ if(1 != streamExpressions.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting a single stream but found %d",expression, streamExpressions.size()));
+ }
+
+ init(tupleStream, id, runInterval, queueSize);
+ }
+
+ public DaemonStream(TupleStream tupleStream, String id, long runInterval, int queueSize) {
+ init(tupleStream, id, runInterval, queueSize);
+ }
+
+ @Override
+ public StreamExpression toExpression(StreamFactory factory) throws IOException {
+ // function name
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+
+ // streams
+ if(tupleStream instanceof Expressible){
+ expression.addParameter(((Expressible)tupleStream).toExpression(factory));
+ } else {
+ throw new IOException("This UniqueStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+ }
+
+ expression.addParameter(new StreamExpressionNamedParameter("id", id));
+ expression.addParameter(new StreamExpressionNamedParameter("runInterval", Long.toString(runInterval)));
+ expression.addParameter(new StreamExpressionNamedParameter("queueSize", Integer.toString(queueSize)));
+
+ return expression;
+ }
+
+ public int remainingCapacity() {
+ return this.queue.remainingCapacity();
+ }
+
+ public void init(TupleStream tupleStream, String id, long runInterval, int queueSize) {
+ this.tupleStream = tupleStream;
+ this.id = id;
+ this.runInterval = runInterval;
+ this.queueSize = queueSize;
+ if(queueSize > 0) {
+ queue = new ArrayBlockingQueue(queueSize);
+ eatTuples = false;
+ } else {
+ eatTuples = true;
+ }
+ }
+
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ public boolean equals(Object o) {
+ if(o instanceof DaemonStream) {
+ return id.equals(((DaemonStream)o).id);
+ }
+ return false;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void open() {
+ this.streamRunner = new StreamRunner(runInterval);
+ this.streamRunner.start();
+ }
+
+ public Tuple read() throws IOException {
+ try {
+ return queue.take();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ public StreamComparator getStreamSort() {
+ return tupleStream.getStreamSort();
+ }
+
+ public void setStreamContext(StreamContext streamContext) {
+ this.tupleStream.setStreamContext(streamContext);
+ }
+
+ public void close() {
+ if(closed) {
+ return;
+ }
+ streamRunner.setShutdown(true);
+ }
+
+ public List<TupleStream> children() {
+ List<TupleStream> children = new ArrayList();
+ children.add(tupleStream);
+ return children;
+ }
+
+ public synchronized Tuple getInfo() {
+ Tuple tuple = new Tuple(new HashMap());
+ tuple.put("id", id);
+ tuple.put("startTime", startTime);
+ tuple.put("stopTime", stopTime);
+ tuple.put("iterations", iterations);
+ tuple.put("state", streamRunner.getState().toString());
+ if(exception != null) {
+ tuple.put("exception", exception.getMessage());
+ }
+
+ return tuple;
+ }
+
+ private synchronized void incrementIterations() {
+ ++iterations;
+ }
+
+ private synchronized void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ private synchronized void setStopTime(long stopTime) {
+ this.stopTime = stopTime;
+ }
+
+ private class StreamRunner extends Thread {
+
+ private long sleepMillis = 1000;
+ private long runInterval;
+ private long lastRun;
+
+ private boolean shutdown;
+
+ public StreamRunner(long runInterval) {
+ this.runInterval = runInterval;
+ }
+
+ public synchronized void setShutdown(boolean shutdown) {
+ this.shutdown = shutdown;
+ interrupt(); //We could be blocked on the queue or sleeping
+ }
+
+ public synchronized boolean getShutdown() {
+ return shutdown;
+ }
+
+ public void run() {
+ setStartTime(new Date().getTime());
+ OUTER:
+ while (!getShutdown()) {
+ long now = new Date().getTime();
+ if((now-lastRun) > this.runInterval) {
+ lastRun = now;
+ try {
+ tupleStream.open();
+ INNER:
+ while (true) {
+ Tuple tuple = tupleStream.read();
+ if (tuple.EOF) {
+ break INNER;
+ } else if (!eatTuples) {
+ try {
+ queue.put(tuple);
+ } catch(InterruptedException e) {
+ break OUTER;
+ }
+ }
+ }
+ } catch (IOException e) {
+ exception = e;
+ logger.error("Error in DaemonStream", e);
+ break OUTER;
+ } finally {
+ try {
+ tupleStream.close();
+ } catch (IOException e1) {
+ if (exception == null) {
+ exception = e1;
+ logger.error("Error in DaemonStream", e1);
+ break OUTER;
+ }
+ }
+ }
+ }
+ incrementIterations();
+ try {
+ Thread.sleep(sleepMillis);
+ } catch (InterruptedException e) {
+ logger.error("Error in DaemonStream", e);
+ break OUTER;
+ }
+ }
+
+ if(!eatTuples) {
+ Map m = new HashMap();
+ m.put("EOF", true);
+ Tuple tuple = new Tuple(m);
+ queue.offer(tuple);
+ }
+ setStopTime(new Date().getTime());
+ }
+ }
+}
\ No newline at end of file
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RollupStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RollupStream.java?rev=1726291&r1=1726290&r2=1726291&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RollupStream.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RollupStream.java Fri Jan 22 19:07:15 2016
@@ -162,6 +162,9 @@ public class RollupStream extends TupleS
public void close() throws IOException {
tupleStream.close();
+ this.currentMetrics = null;
+ this.currentKey = new HashKey("-");
+ this.finished = false;
}
public Tuple read() throws IOException {
Modified: lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java?rev=1726291&r1=1726290&r2=1726291&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java (original)
+++ lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java Fri Jan 22 19:07:15 2016
@@ -20,6 +20,7 @@ package org.apache.solr.client.solrj.io.
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -46,6 +47,7 @@ import org.apache.solr.client.solrj.io.s
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -134,6 +136,7 @@ public class StreamExpressionTest extend
testUniqueStream();
testRollupStream();
testStatsStream();
+ testDaemonStream();
testParallelUniqueStream();
testParallelReducerStream();
testParallelRankStream();
@@ -148,6 +151,7 @@ public class StreamExpressionTest extend
testSubFacetStream();
testUpdateStream();
testParallelUpdateStream();
+ testParallelDaemonUpdateStream();
testIntersectStream();
testParallelIntersectStream();
testComplementStream();
@@ -525,6 +529,130 @@ public class StreamExpressionTest extend
commit();
}
+ private void testDaemonStream() throws Exception {
+
+ indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1");
+ indexr(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2");
+ indexr(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3");
+ indexr(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4");
+ indexr(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5");
+ indexr(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6");
+ indexr(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7");
+ indexr(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8");
+ indexr(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9");
+ indexr(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10");
+
+ commit();
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost("collection1", zkServer.getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("rollup", RollupStream.class)
+ .withFunctionName("sum", SumMetric.class)
+ .withFunctionName("min", MinMetric.class)
+ .withFunctionName("max", MaxMetric.class)
+ .withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("count", CountMetric.class)
+ .withFunctionName("daemon", DaemonStream.class);
+
+ StreamExpression expression;
+ DaemonStream daemonStream;
+
+ expression = StreamExpressionParser.parse("daemon(rollup("
+ + "search(collection1, q=*:*, fl=\"a_i,a_s\", sort=\"a_s asc\"),"
+ + "over=\"a_s\","
+ + "sum(a_i)"
+ + "), id=\"test\", runInterval=\"1000\", queueSize=\"9\")");
+ daemonStream = (DaemonStream)factory.constructStream(expression);
+
+
+ //Test Long and Double Sums
+
+ daemonStream.open(); // This will start the daemon thread
+
+ for(int i=0; i<4; i++) {
+ Tuple tuple = daemonStream.read(); // Reads from the queue
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 1:"+bucket);
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 17.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 2:"+bucket);
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ //System.out.println("#################################### Bucket 3:"+bucket);
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ }
+
+ //Now lets wait until the internal queue fills up
+
+ while(daemonStream.remainingCapacity() > 0) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+
+ }
+ }
+
+ //OK capacity is full, let's index a new doc
+
+ indexr(id, "10", "a_s", "hello0", "a_i", "1", "a_f", "10");
+ commit();
+
+ //Now lets clear the existing docs in the queue 9, plus 3 more to get passed the run that was blocked. The next run should
+ //have the tuples with the updated count.
+ for(int i=0; i<12;i++) {
+ daemonStream.read();
+ }
+
+ //And rerun the loop. It should have a new count for hello0
+ for(int i=0; i<4; i++) {
+ Tuple tuple = daemonStream.read(); // Reads from the queue
+ String bucket = tuple.getString("a_s");
+ Double sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 1:"+bucket);
+ assertTrue(bucket.equals("hello0"));
+ assertTrue(sumi.doubleValue() == 18.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+
+ //System.out.println("#################################### Bucket 2:"+bucket);
+ assertTrue(bucket.equals("hello3"));
+ assertTrue(sumi.doubleValue() == 38.0D);
+
+ tuple = daemonStream.read();
+ bucket = tuple.getString("a_s");
+ sumi = tuple.getDouble("sum(a_i)");
+ //System.out.println("#################################### Bucket 3:"+bucket);
+ assertTrue(bucket.equals("hello4"));
+ assertTrue(sumi.longValue() == 15);
+ }
+
+ daemonStream.close(); //This should stop the daemon thread
+
+ del("*:*");
+ commit();
+ }
+
+
+
+
+
+
private void testRollupStream() throws Exception {
indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1");
@@ -2206,6 +2334,174 @@ public class StreamExpressionTest extend
del("*:*");
commit();
}
+
+ private void testParallelDaemonUpdateStream() throws Exception {
+ CloudSolrClient destinationCollectionClient = createCloudClient("parallelDestinationCollection1");
+ createCollection("parallelDestinationCollection1", destinationCollectionClient, 2, 2);
+
+ indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0", "s_multi", "aaaa", "s_multi", "bbbb", "i_multi", "4", "i_multi", "7");
+ indexr(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0", "s_multi", "aaaa1", "s_multi", "bbbb1", "i_multi", "44", "i_multi", "77");
+ indexr(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "s_multi", "aaaa2", "s_multi", "bbbb2", "i_multi", "444", "i_multi", "777");
+ indexr(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "s_multi", "aaaa3", "s_multi", "bbbb3", "i_multi", "4444", "i_multi", "7777");
+ indexr(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1", "s_multi", "aaaa4", "s_multi", "bbbb4", "i_multi", "44444", "i_multi", "77777");
+ commit();
+ waitForRecoveriesToFinish("parallelDestinationCollection1", false);
+
+ StreamExpression expression;
+ TupleStream stream;
+ Tuple t;
+
+ String zkHost = zkServer.getZkAddress();
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost("collection1", zkServer.getZkAddress())
+ .withCollectionZkHost("parallelDestinationCollection1", zkServer.getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class)
+ .withFunctionName("update", UpdateStream.class)
+ .withFunctionName("parallel", ParallelStream.class)
+ .withFunctionName("daemon", DaemonStream.class);
+
+ //Copy all docs to destinationCollection
+ String updateExpression = "daemon(update(parallelDestinationCollection1, batchSize=2, search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\")), runInterval=\"1000\", id=\"test\")";
+ TupleStream parallelUpdateStream = factory.constructStream("parallel(collection1, " + updateExpression + ", workers=\"2\", zkHost=\""+zkHost+"\", sort=\"batchNumber asc\")");
+ List<Tuple> tuples = getTuples(parallelUpdateStream);
+ assert(tuples.size() == 2);
+
+ //Lets sleep long enough for daemon updates to run.
+ //Lets stop the daemons
+ Map params = new HashMap();
+ params.put(CommonParams.QT,"/stream");
+ params.put("action","list");
+
+ int workersComplete = 0;
+ for(CloudJettyRunner jetty : this.cloudJettys) {
+ int iterations = 0;
+ INNER:
+ while(iterations == 0) {
+ SolrStream solrStream = new SolrStream(jetty.url, params);
+ solrStream.open();
+ Tuple tupleResponse = solrStream.read();
+ if (tupleResponse.EOF) {
+ solrStream.close();
+ break INNER;
+ } else {
+ long l = tupleResponse.getLong("iterations");
+ if(l > 0) {
+ ++workersComplete;
+ } else {
+ try {
+ Thread.sleep(1000);
+ } catch(Exception e) {
+
+ }
+ }
+ iterations = (int) l;
+ solrStream.close();
+ }
+ }
+ }
+
+ assert(workersComplete == 2);
+
+ destinationCollectionClient.commit();
+
+ //Lets stop the daemons
+ params = new HashMap();
+ params.put(CommonParams.QT,"/stream");
+ params.put("action", "stop");
+ params.put("id", "test");
+ for(CloudJettyRunner jetty : this.cloudJettys) {
+ SolrStream solrStream = new SolrStream(jetty.url, params);
+ solrStream.open();
+ Tuple tupleResponse = solrStream.read();
+ solrStream.close();
+ }
+
+ params = new HashMap();
+ params.put(CommonParams.QT,"/stream");
+ params.put("action","list");
+
+ workersComplete = 0;
+ for(CloudJettyRunner jetty : this.cloudJettys) {
+ long stopTime = 0;
+ INNER:
+ while(stopTime == 0) {
+ SolrStream solrStream = new SolrStream(jetty.url, params);
+ solrStream.open();
+ Tuple tupleResponse = solrStream.read();
+ if (tupleResponse.EOF) {
+ solrStream.close();
+ break INNER;
+ } else {
+ stopTime = tupleResponse.getLong("stopTime");
+ if (stopTime > 0) {
+ ++workersComplete;
+ } else {
+ try {
+ Thread.sleep(1000);
+ } catch(Exception e) {
+
+ }
+ }
+ solrStream.close();
+ }
+ }
+ }
+
+ assertTrue(workersComplete == 2);
+ //Ensure that destinationCollection actually has the new docs.
+ expression = StreamExpressionParser.parse("search(parallelDestinationCollection1, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_i asc\")");
+ stream = new CloudSolrStream(expression, factory);
+ tuples = getTuples(stream);
+ assertEquals(5, tuples.size());
+
+ Tuple tuple = tuples.get(0);
+ assert(tuple.getLong("id") == 0);
+ assert(tuple.get("a_s").equals("hello0"));
+ assert(tuple.getLong("a_i") == 0);
+ assert(tuple.getDouble("a_f") == 0.0);
+ assertList(tuple.getStrings("s_multi"), "aaaa", "bbbb");
+ assertList(tuple.getLongs("i_multi"), Long.parseLong("4"), Long.parseLong("7"));
+
+ tuple = tuples.get(1);
+ assert(tuple.getLong("id") == 1);
+ assert(tuple.get("a_s").equals("hello1"));
+ assert(tuple.getLong("a_i") == 1);
+ assert(tuple.getDouble("a_f") == 1.0);
+ assertList(tuple.getStrings("s_multi"), "aaaa4", "bbbb4");
+ assertList(tuple.getLongs("i_multi"), Long.parseLong("44444"), Long.parseLong("77777"));
+
+ tuple = tuples.get(2);
+ assert(tuple.getLong("id") == 2);
+ assert(tuple.get("a_s").equals("hello2"));
+ assert(tuple.getLong("a_i") == 2);
+ assert(tuple.getDouble("a_f") == 0.0);
+ assertList(tuple.getStrings("s_multi"), "aaaa1", "bbbb1");
+ assertList(tuple.getLongs("i_multi"), Long.parseLong("44"), Long.parseLong("77"));
+
+ tuple = tuples.get(3);
+ assert(tuple.getLong("id") == 3);
+ assert(tuple.get("a_s").equals("hello3"));
+ assert(tuple.getLong("a_i") == 3);
+ assert(tuple.getDouble("a_f") == 3.0);
+ assertList(tuple.getStrings("s_multi"), "aaaa2", "bbbb2");
+ assertList(tuple.getLongs("i_multi"), Long.parseLong("444"), Long.parseLong("777"));
+
+ tuple = tuples.get(4);
+ assert(tuple.getLong("id") == 4);
+ assert(tuple.get("a_s").equals("hello4"));
+ assert(tuple.getLong("a_i") == 4);
+ assert(tuple.getDouble("a_f") == 4.0);
+ assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3");
+ assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777"));
+
+ destinationCollectionClient.deleteByQuery("*:*");
+ destinationCollectionClient.commit();
+ destinationCollectionClient.close();
+ del("*:*");
+ commit();
+ }
+
+
private void testIntersectStream() throws Exception{
indexr(id, "0", "a_s", "setA", "a_i", "0");
Modified: lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java?rev=1726291&r1=1726290&r2=1726291&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java (original)
+++ lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java Fri Jan 22 19:07:15 2016
@@ -61,6 +61,7 @@ public class StreamExpressionToExpession
.withFunctionName("min", MinMetric.class)
.withFunctionName("max", MaxMetric.class)
.withFunctionName("avg", MeanMetric.class)
+ .withFunctionName("daemon", DaemonStream.class)
;
}
@@ -102,6 +103,24 @@ public class StreamExpressionToExpession
assertTrue(expressionString.contains("a_s as fieldA"));
}
+
+ @Test
+ public void testDaemonStream() throws Exception {
+
+ DaemonStream stream;
+ String expressionString;
+
+ // Basic test
+ stream = new DaemonStream(StreamExpressionParser.parse("daemon(search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), id=\"blah\", runInterval=\"1000\", queueSize=\"100\")"), factory);
+ expressionString = stream.toExpression(factory).toString();
+ assertTrue(expressionString.contains("daemon(search(collection1,"));
+ assertTrue(expressionString.contains("q=\"*:*\""));
+ assertTrue(expressionString.contains("fl=\"id,a_s,a_i,a_f\""));
+ assertTrue(expressionString.contains("sort=\"a_f asc, a_i asc\""));
+ assertTrue(expressionString.contains("id=blah"));
+ assertTrue(expressionString.contains("queueSize=100"));
+ assertTrue(expressionString.contains("runInterval=1000"));
+ }
@Test
public void testStatsStream() throws Exception {