You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2017/09/23 17:04:28 UTC
[1/3] lucene-solr:master: SOLR-11388: Add monteCarlo Stream Evaluator
to support Monte Carlo simulations WIP
Repository: lucene-solr
Updated Branches:
refs/heads/master d6fa05715 -> d5274ff23
SOLR-11388: Add monteCarlo Stream Evaluator to support Monte Carlo simulations WIP
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5c9af864
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5c9af864
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5c9af864
Branch: refs/heads/master
Commit: 5c9af8640e21c3f7634956636ef4863839cbcd66
Parents: d6fa057
Author: Joel Bernstein <jb...@apache.org>
Authored: Sat Sep 23 12:35:58 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sat Sep 23 13:03:18 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 2 +
.../io/eval/ConstantDistributionEvaluator.java | 44 ++++++++++++
.../solrj/io/eval/MonteCarloEvaluator.java | 76 ++++++++++++++++++++
.../client/solrj/io/eval/SampleEvaluator.java | 30 +++++---
4 files changed, 142 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c9af864/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index bfeb819..0f059e3 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -299,6 +299,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("primes", PrimesEvaluator.class)
.withFunctionName("factorial", FactorialEvaluator.class)
.withFunctionName("movingMedian", MovingMedianEvaluator.class)
+ .withFunctionName("monteCarlo", MonteCarloEvaluator.class)
+ .withFunctionName("constantDistribution", ConstantDistributionEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c9af864/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConstantDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConstantDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConstantDistributionEvaluator.java
new file mode 100644
index 0000000..05a7907
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConstantDistributionEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.ConstantRealDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ConstantDistributionEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
+
+ private static final long serialVersionUID = 1;
+
+ public ConstantDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+
+ Number constant = (Number)first;
+
+ return new ConstantRealDistribution(constant.doubleValue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c9af864/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MonteCarloEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MonteCarloEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MonteCarloEvaluator.java
new file mode 100644
index 0000000..27e806f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MonteCarloEvaluator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class MonteCarloEvaluator extends RecursiveEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public MonteCarloEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ init();
+ }
+
+ public MonteCarloEvaluator(StreamExpression expression, StreamFactory factory, List<String> ignoredNamedParameters) throws IOException{
+ super(expression, factory, ignoredNamedParameters);
+
+ init();
+ }
+
+ private void init() throws IOException{
+ if(2 != containedEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 2 parameters but found %d", toExpression(constructingFactory), containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object evaluate(Tuple tuple) throws IOException {
+ try{
+
+ StreamEvaluator function = containedEvaluators.get(0);
+ StreamEvaluator iterationsEvaluator = containedEvaluators.get(1);
+ Number itNum = (Number)iterationsEvaluator.evaluate(tuple);
+ int it = itNum.intValue();
+ List<Number> results = new ArrayList();
+ for(int i=0; i<it; i++) {
+ Number result = (Number)function.evaluate(tuple);
+ results.add(result);
+ }
+
+ return results;
+ }
+ catch(UncheckedIOException e){
+ throw e.getCause();
+ }
+ }
+
+ @Override
+ public Object doWork(Object... values) throws IOException {
+ // Nothing to do here
+ throw new IOException("This call should never occur");
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c9af864/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
index be98305..1eccebb 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
@@ -27,7 +27,7 @@ import org.apache.commons.math3.distribution.RealDistribution;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-public class SampleEvaluator extends RecursiveObjectEvaluator implements TwoValueWorker {
+public class SampleEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
private static final long serialVersionUID = 1;
@@ -36,26 +36,36 @@ public class SampleEvaluator extends RecursiveObjectEvaluator implements TwoValu
}
@Override
- public Object doWork(Object first, Object second) throws IOException{
- if(null == first){
+ public Object doWork(Object ... objects) throws IOException{
+ if(objects.length < 1){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
}
- if(null == second){
- throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
- }
+
+ Object first = objects[0];
+
if(!(first instanceof RealDistribution) && !(first instanceof IntegerDistribution)){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a Real or Integer Distribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
}
- if(!(second instanceof Number)){
- throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a Number",toExpression(constructingFactory), first.getClass().getSimpleName()));
+
+ Object second = null;
+ if(objects.length > 1) {
+ second = objects[2];
}
if(first instanceof RealDistribution) {
RealDistribution realDistribution = (RealDistribution) first;
- return Arrays.stream(realDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ if(second != null) {
+ return Arrays.stream(realDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ } else {
+ return realDistribution.sample();
+ }
} else {
IntegerDistribution integerDistribution = (IntegerDistribution) first;
- return Arrays.stream(integerDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ if(second != null) {
+ return Arrays.stream(integerDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ } else {
+ return integerDistribution.sample();
+ }
}
}
}
\ No newline at end of file
[2/3] lucene-solr:master: SOLR-11388: Fix sample function
Posted by jb...@apache.org.
SOLR-11388: Fix sample function
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e62f9f85
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e62f9f85
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e62f9f85
Branch: refs/heads/master
Commit: e62f9f85d094ba57e9552904b51653fd2e316116
Parents: 5c9af86
Author: Joel Bernstein <jb...@apache.org>
Authored: Sat Sep 23 12:47:52 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sat Sep 23 13:03:19 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e62f9f85/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
index 1eccebb..ca35a7d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
@@ -49,7 +49,7 @@ public class SampleEvaluator extends RecursiveObjectEvaluator implements ManyVal
Object second = null;
if(objects.length > 1) {
- second = objects[2];
+ second = objects[1];
}
if(first instanceof RealDistribution) {
[3/3] lucene-solr:master: SOLR-11392:
StreamExpressionTest.testParallelExecutorStream fails too frequently
Posted by jb...@apache.org.
SOLR-11392: StreamExpressionTest.testParallelExecutorStream fails too frequently
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d5274ff2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d5274ff2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d5274ff2
Branch: refs/heads/master
Commit: d5274ff239772d1b1c28775cec14abb44e4d80de
Parents: e62f9f8
Author: Joel Bernstein <jb...@apache.org>
Authored: Sat Sep 23 13:00:00 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sat Sep 23 13:03:19 2017 -0400
----------------------------------------------------------------------
.../client/solrj/io/stream/StreamExpressionTest.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5274ff2/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 241dfc2..ded65b8 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -7169,13 +7169,13 @@ public class StreamExpressionTest extends SolrCloudTestCase {
@Test
public void testExecutorStream() throws Exception {
- CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("workQueue", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);
- CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("mainCorpus", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);
- CollectionAdminRequest.createCollection("destination", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("destination", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("destination", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);
@@ -7238,13 +7238,13 @@ public class StreamExpressionTest extends SolrCloudTestCase {
@Test
public void testParallelExecutorStream() throws Exception {
- CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).processAndWait(cluster.getSolrClient(),DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("workQueue", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);
- CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("mainCorpus", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);
- CollectionAdminRequest.createCollection("destination", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("destination", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
AbstractDistribZkTestBase.waitForRecoveriesToFinish("destination", cluster.getSolrClient().getZkStateReader(),
false, true, TIMEOUT);