You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2018/11/07 13:34:21 UTC
lucene-solr:branch_7x: SOLR-12971: Add pivot Stream Evaluator to
pivot facet co-occurrence counts into a matrix
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x decfd6707 -> 8af39cf05
SOLR-12971: Add pivot Stream Evaluator to pivot facet co-occurrence counts into a matrix
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8af39cf0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8af39cf0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8af39cf0
Branch: refs/heads/branch_7x
Commit: 8af39cf055245f68e5bcb5cd89d064333c6b1db4
Parents: decfd67
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Nov 7 08:12:53 2018 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Nov 7 08:30:22 2018 -0500
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 2 +
.../client/solrj/io/eval/PivotEvaluator.java | 111 +++++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 44 ++++++++
4 files changed, 158 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 1777467..200f07b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -273,6 +273,8 @@ public class Lang {
.withFunctionName("getSupportPoints", GetSupportPointsEvaluator.class)
.withFunctionName("pairSort", PairSortEvaluator.class)
.withFunctionName("recip", RecipEvaluator.class)
+ .withFunctionName("pivot", PivotEvaluator.class)
+
// Boolean Stream Evaluators
.withFunctionName("and", AndEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
new file mode 100644
index 0000000..919db14
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Locale;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.Tuple;
+
+public class PivotEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public PivotEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(4 != containedEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 4 values but found %d",expression,containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object[] values) throws IOException {
+ if(values.length != 4) {
+ throw new IOException("The pivot function requires four parameters.");
+ }
+
+ Object value1 = values[0];
+ Object value2 = values[1];
+ Object value3 = values[2];
+ Object value4 = values[3];
+
+ if(value1 instanceof List) {
+ List<Tuple> tuples = (List<Tuple>)value1;
+ String x = (String)value2;
+ x = x.replace("\"", "");
+ String y = (String)value3;
+ y = y.replace("\"", "");
+
+ String vlabel = (String)value4;
+ vlabel = vlabel.replace("\"", "");
+
+ Set<String> xset = new TreeSet();
+ Set<String> yset = new TreeSet();
+
+ for(int i=0; i<tuples.size(); i++) {
+ Tuple tuple = tuples.get(i);
+ xset.add(tuple.getString(x));
+ yset.add(tuple.getString(y));
+ }
+
+ double[][] data = new double[xset.size()][yset.size()];
+
+ List<String> xlabels = new ArrayList(xset.size());
+ Map<String, Integer> xindexes = new HashMap();
+ int xindex = 0;
+ for (String xlabel :xset) {
+ xlabels.add(xlabel);
+ xindexes.put(xlabel, xindex);
+ ++xindex;
+ }
+
+ List<String> ylabels = new ArrayList(yset.size());
+ Map<String, Integer> yindexes = new HashMap();
+ int yindex = 0;
+ for (String ylabel : yset) {
+ ylabels.add(ylabel);
+ yindexes.put(ylabel, yindex);
+ ++yindex;
+ }
+
+ for(Tuple tuple : tuples) {
+ String xlabel = tuple.getString(x);
+ String ylabel = tuple.getString(y);
+ int xi = xindexes.get(xlabel);
+ int yi = yindexes.get(ylabel);
+ double val = tuple.getDouble(vlabel);
+ data[xi][yi] = val;
+ }
+
+ Matrix matrix = new Matrix(data);
+ matrix.setRowLabels(xlabels);
+ matrix.setColumnLabels(ylabels);
+ return matrix;
+ } else {
+ throw new IOException("The getValue function expects a list of tuples as the first parameter");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 960eb50..98a6d45 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -73,7 +73,7 @@ public class TestLang extends LuceneTestCase {
"outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "latlonVectors",
"convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize","oscillate",
"getAmplitude", "getPhase", "getAngularFrequency", "enclosingDisk", "getCenter", "getRadius",
- "getSupportPoints", "pairSort", "log10", "plist", "recip"};
+ "getSupportPoints", "pairSort", "log10", "plist", "recip", "pivot"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 4642388..9c93ced 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -2164,6 +2164,50 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testPivot() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=list(tuple(fx=x1, fy=f1, fv=add(1,1)), " +
+ " tuple(fx=x1, fy=f2, fv=add(1,3)), " +
+ " tuple(fx=x2, fy=f1, fv=add(1,7)), " +
+ " tuple(fx=x3, fy=f1, fv=add(1,4))," +
+ " tuple(fx=x3, fy=f3, fv=add(1,7)))," +
+ " b=pivot(a, fx, fy, fv)," +
+ " c=getRowLabels(b)," +
+ " d=getColumnLabels(b))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertEquals(tuples.size(), 1);
+ List<List<Number>> matrix = (List<List<Number>>)tuples.get(0).get("b");
+ List<Number> row1 = matrix.get(0);
+ assertEquals(row1.get(0).doubleValue(), 2.0,0);
+ assertEquals(row1.get(1).doubleValue(), 4.0,0);
+ assertEquals(row1.get(2).doubleValue(), 0,0);
+ List<Number> row2 = matrix.get(1);
+ assertEquals(row2.get(0).doubleValue(), 8.0,0);
+ assertEquals(row2.get(1).doubleValue(), 0,0);
+ assertEquals(row2.get(2).doubleValue(), 0,0);
+ List<Number> row3 = matrix.get(2);
+ assertEquals(row3.get(0).doubleValue(), 5.0,0);
+ assertEquals(row3.get(1).doubleValue(), 0,0);
+ assertEquals(row3.get(2).doubleValue(), 8.0,0);
+
+ List<String> rowLabels = (List<String>)tuples.get(0).get("c");
+ assertEquals(rowLabels.get(0), "x1");
+ assertEquals(rowLabels.get(1), "x2");
+ assertEquals(rowLabels.get(2), "x3");
+ List<String> columnLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnLabels.get(0), "f1");
+ assertEquals(columnLabels.get(1), "f2");
+ assertEquals(columnLabels.get(2), "f3");
+ }
+
+ @Test
public void testEbeSubtract() throws Exception {
String cexpr = "let(echo=true," +
" a=array(2, 4, 6, 8, 10, 12)," +