You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2020/05/28 21:11:18 UTC
[incubator-pinot] branch master updated: Include transformFunction
in timeFieldSpec toJsonObject (#5464)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 94aad60 Include transformFunction in timeFieldSpec toJsonObject (#5464)
94aad60 is described below
commit 94aad60f520d627e257cac6c57c54abeafec37a2
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Thu May 28 14:11:08 2020 -0700
Include transformFunction in timeFieldSpec toJsonObject (#5464)
---
.../apache/pinot/common/data/FieldSpecTest.java | 10 ++++++---
pinot-common/src/test/resources/schemaTest.schema | 25 +++++++++++++++++++---
.../java/org/apache/pinot/spi/data/FieldSpec.java | 10 ++++++---
.../org/apache/pinot/spi/data/TimeFieldSpec.java | 1 +
4 files changed, 37 insertions(+), 9 deletions(-)
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
index 9d29bd4..8063988 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
@@ -262,7 +262,7 @@ public class FieldSpecTest {
FieldSpec second;
// Single-value boolean type dimension field with default null value.
- String[] dimensionFields = {"\"name\":\"dimension\"", "\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false"};
+ String[] dimensionFields = {"\"name\":\"dimension\"", "\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false", "\"transformFunction\":\"trim(foo)\""};
first = JsonUtils.stringToObject(getRandomOrderJsonString(dimensionFields), DimensionFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(), DimensionFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
@@ -276,14 +276,18 @@ public class FieldSpecTest {
// Time field with default value.
String[] timeFields =
- {"\"incomingGranularitySpec\":{\"timeUnitSize\":1, \"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}", "\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}", "\"defaultNullValue\":-1"};
+ {"\"incomingGranularitySpec\":{\"timeUnitSize\":1, \"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}",
+ "\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}",
+ "\"defaultNullValue\":-1",
+ "\"transformFunction\":\"toEpochDays(millis)\""};
first = JsonUtils.stringToObject(getRandomOrderJsonString(timeFields), TimeFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(), TimeFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
// DateTime field
String[] dateTimeFields =
- {"\"name\":\"Date\"", "\"dataType\":\"LONG\"", "\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"", "\"dateTimeType\":\"PRIMARY\""};
+ {"\"name\":\"Date\"", "\"dataType\":\"LONG\"", "\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"",
+ "\"transformFunction\":\"fromEpochDays(daysSinceEpoch)\""};
first = JsonUtils.stringToObject(getRandomOrderJsonString(dateTimeFields), DateTimeFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(), DateTimeFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
diff --git a/pinot-common/src/test/resources/schemaTest.schema b/pinot-common/src/test/resources/schemaTest.schema
index 32a83c0..b7c7dcf 100644
--- a/pinot-common/src/test/resources/schemaTest.schema
+++ b/pinot-common/src/test/resources/schemaTest.schema
@@ -8,7 +8,8 @@
{
"name": "longMetric",
"dataType": "LONG",
- "singleValueField": true
+ "singleValueField": true,
+ "transformFunction": "Groovy({foo/1000}, foo)"
},
{
"name": "floatMetric",
@@ -37,7 +38,8 @@
"name": "longDimension",
"dataType": "LONG",
"singleValueField": false,
- "delimiter": null
+ "delimiter": null,
+ "transformFunction": "trim(bar)"
},
{
"name": "floatDimension",
@@ -69,7 +71,24 @@
"timeType": "DAYS",
"name": "outgoing"
},
- "defaultNullValue": 10000
+ "defaultNullValue": 10000,
+ "transformFunction": "toEpochDays(millis)"
},
+ "dateTimeFieldSpecs": [
+ {
+ "name": "dateTime1",
+ "dataType": "LONG",
+ "format": "1:MILLISECONDS:EPOCH",
+ "granularity": "15:MINUTES",
+ "transformFunction": "fromDateTime(dt, 'yyyyMMdd')"
+ },
+ {
+ "name": "dateTime2",
+ "dataType": "STRING",
+ "format": "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd",
+ "granularity": "1:DAYS",
+ "defaultNullValue": "20200101"
+ }
+ ],
"schemaName": "schemaTest"
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
index de3e3ee..aabbd7f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
@@ -263,9 +263,7 @@ public abstract class FieldSpec implements Comparable<FieldSpec> {
jsonObject.put("maxLength", _maxLength);
}
appendDefaultNullValue(jsonObject);
- if (_transformFunction != null) {
- jsonObject.put("transformFunction", _transformFunction);
- }
+ appendTransformFunction(jsonObject);
return jsonObject;
}
@@ -280,6 +278,12 @@ public abstract class FieldSpec implements Comparable<FieldSpec> {
}
}
+ protected void appendTransformFunction(ObjectNode jsonNode) {
+ if (_transformFunction != null) {
+ jsonNode.put("transformFunction", _transformFunction);
+ }
+ }
+
@SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
@Override
public boolean equals(Object o) {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
index 5fefbf7..1699ab8 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
@@ -112,6 +112,7 @@ public final class TimeFieldSpec extends FieldSpec {
jsonObject.set("outgoingGranularitySpec", _outgoingGranularitySpec.toJsonObject());
}
appendDefaultNullValue(jsonObject);
+ appendTransformFunction(jsonObject);
return jsonObject;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org