You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2020/05/14 14:30:43 UTC
[lucene-solr] branch branch_8x updated (f00b38d -> 081f1ec)
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a change to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from f00b38d SOLR-14351: commitScheduler was missing MDC logging (#1498)
new 8e699e1 SOLR-14407: Handle shards.purpose in the postlogs tool
new 081f1ec SOLR-14478: Allow the diff Stream Evaluator to operate on the rows of a matrix
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/solr/util/SolrLogPostTool.java | 69 +++++++++++++++-
.../org/apache/solr/util/SolrLogPostToolTest.java | 6 +-
.../solrj/io/eval/TimeDifferencingEvaluator.java | 96 +++++++++++++++++-----
.../client/solrj/io/stream/MathExpressionTest.java | 36 ++++++++
4 files changed, 184 insertions(+), 23 deletions(-)
[lucene-solr] 01/02: SOLR-14407: Handle shards.purpose in the
postlogs tool
Posted by jb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 8e699e1fb03c213c8673b903815ac414517eefab
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Wed May 13 09:56:53 2020 -0400
SOLR-14407: Handle shards.purpose in the postlogs tool
---
.../java/org/apache/solr/util/SolrLogPostTool.java | 69 +++++++++++++++++++++-
.../org/apache/solr/util/SolrLogPostToolTest.java | 6 +-
2 files changed, 73 insertions(+), 2 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
index 24a2ba6..715996f 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
@@ -18,9 +18,12 @@ package org.apache.solr.util;
import java.io.*;
import java.nio.charset.Charset;
+import java.util.Collections;
import java.util.List;
import java.util.ArrayList;
import java.net.URLDecoder;
+import java.util.Map;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
@@ -29,6 +32,8 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
+import org.apache.solr.handler.component.ShardRequest;
+
/**
@@ -309,6 +314,7 @@ public class SolrLogPostTool {
return doc;
}
+
private SolrInputDocument parseNewSearch(String line) {
SolrInputDocument doc = new SolrInputDocument();
@@ -514,13 +520,25 @@ public class SolrLogPostTool {
} catch (Exception e) {
throw new IOException(e);
}
- }
+ if(parts[0].equals("shards.purpose")) {
+ try {
+ int purpose = Integer.parseInt(parts[1]);
+ String[] purposes = getRequestPurposeNames(purpose);
+ for (String p : purposes) {
+ doc.addField("purpose_ss", p);
+ }
+ } catch(Throwable e) {
+ //We'll just sit on this for now and not interrupt the load for this one field.
+ }
+ }
+ }
//Special params used to determine what stage a query is.
//So we populate with defaults.
//The absence of the distrib params means its a distributed query.
+
if(doc.getField("distrib_s") == null) {
doc.addField("distrib_s", "true");
}
@@ -542,4 +560,53 @@ public class SolrLogPostTool {
return "/get".equals(path.getValue());
}
}
+
+ private static final Map<Integer, String> purposes;
+ protected static final String UNKNOWN_VALUE = "Unknown";
+ private static final String[] purposeUnknown = new String[] { UNKNOWN_VALUE };
+
+ public static String[] getRequestPurposeNames(Integer reqPurpose) {
+ if (reqPurpose != null) {
+ int valid = 0;
+ for (Map.Entry<Integer, String>entry : purposes.entrySet()) {
+ if ((reqPurpose & entry.getKey()) != 0) {
+ valid++;
+ }
+ }
+ if (valid == 0) {
+ return purposeUnknown;
+ } else {
+ String[] result = new String[valid];
+ int i = 0;
+ for (Map.Entry<Integer, String>entry : purposes.entrySet()) {
+ if ((reqPurpose & entry.getKey()) != 0) {
+ result[i] = entry.getValue();
+ i++;
+ }
+ }
+ return result;
+ }
+ }
+ return purposeUnknown;
+ }
+
+ static {
+ Map<Integer, String> map = new TreeMap<>();
+ map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE");
+ map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS");
+ map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS");
+ map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS");
+ map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS");
+ map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS");
+ map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS");
+ map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG");
+ map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS");
+ map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS");
+ map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS");
+ map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS");
+ map.put(ShardRequest.PURPOSE_REFINE_PIVOT_FACETS, "REFINE_PIVOT_FACETS");
+ map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS");
+ map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS");
+ purposes = Collections.unmodifiableMap(map);
+ }
}
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
index 9f9322a..10f7b8c 100644
--- a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
@@ -33,7 +33,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
@Test
public void testQueryRecord() throws Exception{
- String record = "2019-12-09 15:05:01.931 INFO (qtp2103763750-21) [c:logs4 s:shard1 r:core_node2 x:logs4_shard1_replica_n1] o.a.s.c.S.Request [logs4_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_=1575835181759&isShard=true&wt=javabin&distrib=false} hits=234868 status=0 QTime=8\n";
+ String record = "2019-12-09 15:05:01.931 INFO (qtp2103763750-21) [c:logs4 s:shard1 r:core_node2 x:logs4_shard1_replica_n1] o.a.s.c.S.Request [logs4_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_=1575835181759&shards.purpose=36&isShard=true&wt=javabin&distrib=false} hits=234868 status=0 QTime=8\n";
List<SolrInputDocument> docs = readDocs(record);
assertEquals(docs.size(), 1);
SolrInputDocument doc = docs.get(0);
@@ -53,6 +53,8 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
SolrInputField isShard = doc.getField("isShard_s");
SolrInputField ids = doc.getField("ids_s");
SolrInputField shards = doc.getField("shards_s");
+ SolrInputField purpose = doc.getField("purpose_ss");
+ Object[] purposes = purpose.getValues().toArray();
assertEquals(query.getValue(), "*:*");
assertEquals(date.getValue(), "2019-12-09T15:05:01.931");
@@ -69,6 +71,8 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(isShard.getValue(), "true");
assertEquals(ids.getValue(), "false");
assertEquals(shards.getValue(), "false");
+ assertEquals("GET_TOP_IDS", purposes[0].toString());
+ assertEquals("REFINE_FACETS", purposes[1].toString());
}
@Test
[lucene-solr] 02/02: SOLR-14478: Allow the diff Stream Evaluator to
operate on the rows of a matrix
Posted by jb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 081f1ec530eec63f9bc29aa7b57eb08c602f2d73
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Wed May 13 13:53:51 2020 -0400
SOLR-14478: Allow the diff Stream Evaluator to operate on the rows of a matrix
---
.../solrj/io/eval/TimeDifferencingEvaluator.java | 96 +++++++++++++++++-----
.../client/solrj/io/stream/MathExpressionTest.java | 36 ++++++++
2 files changed, 111 insertions(+), 21 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TimeDifferencingEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TimeDifferencingEvaluator.java
index 6424d0a..eeb97f6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TimeDifferencingEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TimeDifferencingEvaluator.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
import java.util.List;
+import java.util.ArrayList;
import java.util.Locale;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -26,7 +27,7 @@ import java.util.stream.IntStream;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-public class TimeDifferencingEvaluator extends RecursiveNumericEvaluator implements ManyValueWorker{
+public class TimeDifferencingEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker{
protected static final long serialVersionUID = 1L;
@@ -38,32 +39,85 @@ public class TimeDifferencingEvaluator extends RecursiveNumericEvaluator impleme
}
@Override
public Object doWork(Object... values) throws IOException {
- if (!(1 == values.length || values.length == 2)){
- throw new IOException(String.format(Locale.ROOT,"%s(...) only works with 1 or 2 values but %d were provided", constructingFactory.getFunctionName(getClass()), values.length));
+ if (!(1 == values.length || values.length == 2)) {
+ throw new IOException(String.format(Locale.ROOT, "%s(...) only works with 1 or 2 values but %d were provided", constructingFactory.getFunctionName(getClass()), values.length));
}
- List<Number> timeseriesValues = (List<Number> )values[0];
- Number lagValue = 1;
+ if (values[0] instanceof List) {
+ List<Number> timeseriesValues = (List<Number>) values[0];
+ Number lagValue = 1;
- if(1 == values.length) {
- if (!(timeseriesValues instanceof List<?>)) {
- throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the first value, expecting a List", toExpression(constructingFactory), values[0].getClass().getSimpleName()));
+ if (1 == values.length) {
+ if (!(timeseriesValues instanceof List<?>)) {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the first value, expecting a List", toExpression(constructingFactory), values[0].getClass().getSimpleName()));
+ }
+ if (!(timeseriesValues.size() > 1)) {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found list size of %s for the first value, expecting a List of size > 0.", toExpression(constructingFactory), timeseriesValues.size()));
+ }
}
- if (!(timeseriesValues.size() > 1)) {
- throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found list size of %s for the first value, expecting a List of size > 0.", toExpression(constructingFactory), timeseriesValues.size()));
+ if (2 == values.length) {
+ lagValue = (Number) values[1];
+ if (!(lagValue instanceof Number)) {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the second value, expecting a Number", toExpression(constructingFactory), values[1].getClass().getSimpleName()));
+ }
+ if (lagValue.intValue() > timeseriesValues.size()) {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found a lag size of %s for the second value, the first value has a List size of %s, expecting a lag value less than the List size", toExpression(constructingFactory), lagValue.intValue(), timeseriesValues.size()));
+ }
}
- }
- if(2 == values.length) {
- lagValue = (Number) values[1];
- if(!(lagValue instanceof Number)){
- throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the second value, expecting a Number", toExpression(constructingFactory), values[1].getClass().getSimpleName()));
+ final int lag = lagValue.intValue();
+ return IntStream.range(lag, timeseriesValues.size())
+ .mapToObj(n -> (timeseriesValues.get(n).doubleValue() - timeseriesValues.get(n - lag).doubleValue()))
+ .collect(Collectors.toList());
+ } else if(values[0] instanceof Matrix) {
+
+ //Diff each row of the matrix
+
+ Matrix matrix = (Matrix)values[0];
+ double[][] data = matrix.getData();
+ double[][] diffedData = new double[data.length][];
+ Number lagValue = 1;
+
+ if (2 == values.length) {
+ lagValue = (Number) values[1];
+ if (!(lagValue instanceof Number)) {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the second value, expecting a Number", toExpression(constructingFactory), values[1].getClass().getSimpleName()));
+ }
+ }
+
+ int lag = lagValue.intValue();
+
+ for(int i=0; i<data.length; i++) {
+ double[] row = data[i];
+ List<Double> timeseriesValues = new ArrayList(row.length);
+ for(double d : row) {
+ timeseriesValues.add(d);
+ }
+
+ List<Number> diffedList = IntStream.range(lag, timeseriesValues.size())
+ .mapToObj(n -> (timeseriesValues.get(n).doubleValue() - timeseriesValues.get(n - lag).doubleValue()))
+ .collect(Collectors.toList());
+ double[] diffedRow = new double[diffedList.size()];
+ for(int r=0; r<diffedList.size(); r++) {
+ diffedRow[r] = diffedList.get(r).doubleValue();
+ }
+ diffedData[i] = diffedRow;
}
- if (lagValue.intValue() > timeseriesValues.size()) {
- throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found a lag size of %s for the second value, the first value has a List size of %s, expecting a lag value less than the List size", toExpression(constructingFactory), lagValue.intValue(), timeseriesValues.size()));
+
+ Matrix diffedMatrix = new Matrix(diffedData);
+ diffedMatrix.setRowLabels(matrix.getRowLabels());
+ List<String> columns = matrix.getColumnLabels();
+ if(columns != null) {
+ List<String> newColumns = new ArrayList(columns.size() - lag);
+
+ for (int i = lag; i < columns.size(); i++) {
+ newColumns.add(columns.get(i));
+ }
+
+ diffedMatrix.setColumnLabels(newColumns);
}
+ return diffedMatrix;
+
+ } else {
+ throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - first parameter must be list of matrix", toExpression(constructingFactory)));
}
- final int lag = lagValue.intValue();
- return IntStream.range(lag, timeseriesValues.size())
- .mapToObj(n -> (timeseriesValues.get(n).doubleValue()-timeseriesValues.get(n-lag).doubleValue()))
- .collect(Collectors.toList());
}
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index da71685..82f7443 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -4062,6 +4062,42 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertEquals((double)out.get(20), 22.92, 0.009);
}
+
+ @Test
+ public void testTimeDifferencingMatrix() throws Exception {
+ String cexpr = "let(echo=\"c, d\",\n" +
+ " a=matrix(array(1,2,3,4,5),array(7.5,9,11,15.5,50.2)),\n" +
+ " b=setColumnLabels(a, array(\"a\",\"b\",\"c\",\"d\",\"e\")),\n" +
+ " c=diff(b, 2),\n" +
+ " d=getColumnLabels(c))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>> matrix = (List<List<Number>>)tuples.get(0).get("c");
+ List<String> columnsLabels = (List<String>)tuples.get(0).get("d");
+ assertEquals(columnsLabels.size(), 3);
+ assertEquals(columnsLabels.get(0), "c");
+ assertEquals(columnsLabels.get(1), "d");
+ assertEquals(columnsLabels.get(2), "e");
+ assertEquals(matrix.size(), 2);
+ List<Number> row1 = matrix.get(0);
+ List<Number> row2 = matrix.get(1);
+ assertEquals(row1.size(), 3);
+ assertEquals(row1.get(0).doubleValue(), 2.0, 0);
+ assertEquals(row1.get(1).doubleValue(), 2.0, 0);
+ assertEquals(row1.get(2).doubleValue(), 2.0, 0);
+ assertEquals(row2.size(), 3 );
+ assertEquals(row2.get(0).doubleValue(), 3.5, 0);
+ assertEquals(row2.get(1).doubleValue(), 6.5, 0);
+ assertEquals(row2.get(2).doubleValue(), 39.2, 0);
+ }
+
@Test
public void testTimeDifferencingDefaultLag() throws Exception {
String cexpr = "diff(array(1709.0, 1621.0, 1973.0, 1812.0, 1975.0, 1862.0, 1940.0, 2013.0, 1596.0, 1725.0, 1676.0, 1814.0, 1615.0, 1557.0, 1891.0, 1956.0, 1885.0, 1623.0, 1903.0, 1997.0))";