You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2017/05/30 18:59:00 UTC
[1/2] lucene-solr:master: SOLR-10770: Add date formatting to
timeseries Streaming Expression
Repository: lucene-solr
Updated Branches:
refs/heads/master d14ca98df -> 4608e7d03
SOLR-10770: Add date formatting to timeseries Streaming Expression
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/52076291
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/52076291
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/52076291
Branch: refs/heads/master
Commit: 520762913af97f761377e03139499aeee31d2a9f
Parents: d14ca98
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue May 30 14:38:50 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Tue May 30 14:39:38 2017 -0400
----------------------------------------------------------------------
.../solrj/io/stream/TimeSeriesStream.java | 36 ++++++++-
.../solrj/io/stream/StreamExpressionTest.java | 81 ++++++++++++++++++++
2 files changed, 114 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52076291/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
index bb965b0..f38da85 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
@@ -17,6 +17,10 @@
package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -53,6 +57,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
private String end;
private String gap;
private String field;
+ private DateTimeFormatter formatter;
+ private SimpleDateFormat ISOFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
private Metric[] metrics;
private List<Tuple> tuples = new ArrayList();
@@ -70,8 +76,9 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
String field,
String start,
String end,
- String gap) throws IOException {
- init(collection, params, field, metrics, start, end, gap, zkHost);
+ String gap,
+ String format) throws IOException {
+ init(collection, params, field, metrics, start, end, gap, format, zkHost);
}
public TimeSeriesStream(StreamExpression expression, StreamFactory factory) throws IOException{
@@ -82,9 +89,17 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
StreamExpressionNamedParameter endExpression = factory.getNamedOperand(expression, "end");
StreamExpressionNamedParameter fieldExpression = factory.getNamedOperand(expression, "field");
StreamExpressionNamedParameter gapExpression = factory.getNamedOperand(expression, "gap");
+ StreamExpressionNamedParameter formatExpression = factory.getNamedOperand(expression, "format");
+ StreamExpressionNamedParameter qExpression = factory.getNamedOperand(expression, "q");
+
StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression, "zkHost");
List<StreamExpression> metricExpressions = factory.getExpressionOperandsRepresentingTypes(expression, Expressible.class, Metric.class);
+
+ if(qExpression == null) {
+ throw new IOException("The timeseries expression requires the q parameter");
+ }
+
String start = null;
if(startExpression != null) {
start = ((StreamExpressionValue)startExpression.getParameter()).getValue();
@@ -105,6 +120,11 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
field = ((StreamExpressionValue)fieldExpression.getParameter()).getValue();
}
+ String format = null;
+ if(formatExpression != null) {
+ format = ((StreamExpressionValue)formatExpression.getParameter()).getValue();
+ }
+
// Collection Name
if(null == collectionName){
throw new IOException(String.format(Locale.ROOT,"invalid expression %s - collectionName expected as first operand",expression));
@@ -149,7 +169,7 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
}
// We've got all the required items
- init(collectionName, params, field, metrics, start, end, gap , zkHost);
+ init(collectionName, params, field, metrics, start, end, gap, format, zkHost);
}
public String getCollection() {
@@ -163,6 +183,7 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
String start,
String end,
String gap,
+ String format,
String zkHost) throws IOException {
this.zkHost = zkHost;
this.collection = collection;
@@ -175,6 +196,9 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
this.field = field;
this.params = params;
this.end = end;
+ if(format != null) {
+ formatter = DateTimeFormatter.ofPattern(format);
+ }
}
@Override
@@ -348,6 +372,12 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
for(int b=0; b<allBuckets.size(); b++) {
NamedList bucket = (NamedList)allBuckets.get(b);
Object val = bucket.get("val");
+
+ if(formatter != null) {
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(((java.util.Date) val).toInstant(), ZoneOffset.UTC);
+ val = localDateTime.format(formatter);
+ }
+
Tuple t = currentTuple.clone();
t.put(field, val);
int m = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52076291/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index d82ca2a..3957915 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5368,6 +5368,87 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.get(3).getDouble("max(price_f)").equals(400D));
assertTrue(tuples.get(3).getDouble("min(price_f)").equals(400D));
+
+ expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "format=\"yyyy\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 4);
+
+ assertTrue(tuples.get(0).get("test_dt").equals("2013"));
+ assertTrue(tuples.get(0).getLong("count(*)").equals(100L));
+ assertTrue(tuples.get(0).getDouble("sum(price_f)").equals(10000D));
+ assertTrue(tuples.get(0).getDouble("max(price_f)").equals(100D));
+ assertTrue(tuples.get(0).getDouble("min(price_f)").equals(100D));
+
+ assertTrue(tuples.get(1).get("test_dt").equals("2014"));
+ assertTrue(tuples.get(1).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(1).getDouble("sum(price_f)").equals(25000D));
+ assertTrue(tuples.get(1).getDouble("max(price_f)").equals(500D));
+ assertTrue(tuples.get(1).getDouble("min(price_f)").equals(500D));
+
+ assertTrue(tuples.get(2).get("test_dt").equals("2015"));
+ assertTrue(tuples.get(2).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(2).getDouble("sum(price_f)").equals(15000D));
+ assertTrue(tuples.get(2).getDouble("max(price_f)").equals(300D));
+ assertTrue(tuples.get(2).getDouble("min(price_f)").equals(300D));
+
+ assertTrue(tuples.get(3).get("test_dt").equals("2016"));
+ assertTrue(tuples.get(3).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(3).getDouble("sum(price_f)").equals(20000D));
+ assertTrue(tuples.get(3).getDouble("max(price_f)").equals(400D));
+ assertTrue(tuples.get(3).getDouble("min(price_f)").equals(400D));
+
+ expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "format=\"yyyy-MM\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 4);
+
+ assertTrue(tuples.get(0).get("test_dt").equals("2013-01"));
+ assertTrue(tuples.get(0).getLong("count(*)").equals(100L));
+ assertTrue(tuples.get(0).getDouble("sum(price_f)").equals(10000D));
+ assertTrue(tuples.get(0).getDouble("max(price_f)").equals(100D));
+ assertTrue(tuples.get(0).getDouble("min(price_f)").equals(100D));
+
+ assertTrue(tuples.get(1).get("test_dt").equals("2014-01"));
+ assertTrue(tuples.get(1).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(1).getDouble("sum(price_f)").equals(25000D));
+ assertTrue(tuples.get(1).getDouble("max(price_f)").equals(500D));
+ assertTrue(tuples.get(1).getDouble("min(price_f)").equals(500D));
+
+ assertTrue(tuples.get(2).get("test_dt").equals("2015-01"));
+ assertTrue(tuples.get(2).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(2).getDouble("sum(price_f)").equals(15000D));
+ assertTrue(tuples.get(2).getDouble("max(price_f)").equals(300D));
+ assertTrue(tuples.get(2).getDouble("min(price_f)").equals(300D));
+
+ assertTrue(tuples.get(3).get("test_dt").equals("2016-01"));
+ assertTrue(tuples.get(3).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(3).getDouble("sum(price_f)").equals(20000D));
+ assertTrue(tuples.get(3).getDouble("max(price_f)").equals(400D));
+ assertTrue(tuples.get(3).getDouble("min(price_f)").equals(400D));
+
}
@Test
[2/2] lucene-solr:master: SOLR-10770: Fix precommit
Posted by jb...@apache.org.
SOLR-10770: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4608e7d0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4608e7d0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4608e7d0
Branch: refs/heads/master
Commit: 4608e7d03690755a05f53693519db6b06c6e9d7c
Parents: 5207629
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue May 30 14:51:46 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Tue May 30 14:51:46 2017 -0400
----------------------------------------------------------------------
.../apache/solr/client/solrj/io/stream/TimeSeriesStream.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4608e7d0/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
index f38da85..fe2a106 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
@@ -17,7 +17,6 @@
package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
-import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
@@ -57,8 +56,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
private String end;
private String gap;
private String field;
+ private String format;
private DateTimeFormatter formatter;
- private SimpleDateFormat ISOFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
private Metric[] metrics;
private List<Tuple> tuples = new ArrayList();
@@ -197,7 +196,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
this.params = params;
this.end = end;
if(format != null) {
- formatter = DateTimeFormatter.ofPattern(format);
+ this.format = format;
+ formatter = DateTimeFormatter.ofPattern(format, Locale.ROOT);
}
}
@@ -225,6 +225,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
expression.addParameter(new StreamExpressionNamedParameter("end", end));
expression.addParameter(new StreamExpressionNamedParameter("gap", gap));
expression.addParameter(new StreamExpressionNamedParameter("field", gap));
+ expression.addParameter(new StreamExpressionNamedParameter("format", format));
+
// zkHost
expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));