You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ij...@apache.org on 2017/01/17 01:37:06 UTC
nifi git commit: NIFI-2908 Added TimeZone to toDate() and format() EL
Repository: nifi
Updated Branches:
refs/heads/master ef54a8ec6 -> 3bb876eb8
NIFI-2908 Added TimeZone to toDate() and format() EL
This closes #1381.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3bb876eb
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3bb876eb
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3bb876eb
Branch: refs/heads/master
Commit: 3bb876eb835499845e66848e815cd166b683232c
Parents: ef54a8e
Author: Pierre Villard <pi...@gmail.com>
Authored: Tue Jan 3 13:16:40 2017 +0100
Committer: Koji Kawamura <ij...@apache.org>
Committed: Tue Jan 17 10:36:47 2017 +0900
----------------------------------------------------------------------
.../language/antlr/AttributeExpressionParser.g | 8 ++++----
.../attribute/expression/language/Query.java | 14 ++++++++++---
.../evaluation/functions/FormatEvaluator.java | 17 ++++++++++++++--
.../functions/StringToDateEvaluator.java | 17 ++++++++++++++--
.../expression/language/TestQuery.java | 12 +++++++++--
.../asciidoc/expression-language-guide.adoc | 21 ++++++++++++--------
6 files changed, 68 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
index 576f011..4d48ee0 100644
--- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
@@ -75,10 +75,10 @@ tokens {
// functions that return Strings
zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
- PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
+ PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE) LPAREN! anyArg COMMA! anyArg RPAREN!) |
- (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
+ ((SUBSTRING | FORMAT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!;
// functions that return Booleans
@@ -93,13 +93,13 @@ multiArgBool : (IN) LPAREN! anyArg (COMMA! anyArg)* RPAREN!;
// functions that return Numbers (whole or decimal)
zeroArgNum : (LENGTH | TO_NUMBER | TO_DECIMAL | COUNT) LPAREN! RPAREN!;
oneArgNum : ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) |
- (TO_DATE LPAREN! anyArg? RPAREN!) |
((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!);
oneOrTwoArgNum : MATH LPAREN! anyArg (COMMA! anyArg)? RPAREN!;
+zeroOrOneOrTwoArgNum : TO_DATE LPAREN! anyArg? (COMMA! anyArg)? RPAREN!;
stringFunctionRef : zeroArgString | oneArgString | twoArgString | fiveArgString;
booleanFunctionRef : zeroArgBool | oneArgBool | multiArgBool;
-numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum;
+numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum | zeroOrOneOrTwoArgNum;
anyArg : WHOLE_NUMBER | DECIMAL | numberFunctionRef | STRING_LITERAL | zeroArgString | oneArgString | twoArgString | fiveArgString | booleanLiteral | zeroArgBool | oneArgBool | multiArgBool | expression;
stringArg : STRING_LITERAL | zeroArgString | oneArgString | twoArgString | expression;
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
index 18396aa..4b1ce59 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
@@ -1225,8 +1225,10 @@ public class Query {
case TO_DATE: {
if (argEvaluators.isEmpty()) {
return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
- } else if (subjectEvaluator.getResultType() == ResultType.STRING) {
- return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "toDate");
+ } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 1) {
+ return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), null), "toDate");
+ } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 2) {
+ return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "toDate");
} else {
return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
}
@@ -1310,7 +1312,13 @@ public class Query {
toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf")), "lastIndexOf");
}
case FORMAT: {
- return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format")), "format");
+ if(argEvaluators.size() == 1) {
+ return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format"), null), "format");
+ } else if (argEvaluators.size() == 2) {
+ return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "format");
+ } else {
+ throw new AttributeExpressionLanguageParsingException("format() function takes 1 or 2 arguments");
+ }
}
case OR: {
return addToken(new OrEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "or");
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
index 717cbd5..00ce430 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
@@ -20,6 +20,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
+import java.util.TimeZone;
import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
@@ -31,10 +32,12 @@ public class FormatEvaluator extends StringEvaluator {
private final DateEvaluator subject;
private final Evaluator<String> format;
+ private final Evaluator<String> timeZone;
- public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format) {
+ public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
this.subject = subject;
this.format = format;
+ this.timeZone = timeZone;
}
@Override
@@ -50,7 +53,17 @@ public class FormatEvaluator extends StringEvaluator {
return null;
}
- return new StringQueryResult(new SimpleDateFormat(format, Locale.US).format(subjectValue));
+ final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US);
+
+ if(timeZone != null) {
+ final QueryResult<String> tzResult = timeZone.evaluate(attributes);
+ final String tz = tzResult.getValue();
+ if(tz != null && TimeZone.getTimeZone(tz) != null) {
+ sdf.setTimeZone(TimeZone.getTimeZone(tz));
+ }
+ }
+
+ return new StringQueryResult(sdf.format(subjectValue));
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
index 590176e..387777a 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
@@ -21,6 +21,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
+import java.util.TimeZone;
import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
@@ -32,10 +33,12 @@ public class StringToDateEvaluator extends DateEvaluator {
private final Evaluator<String> subject;
private final Evaluator<String> format;
+ private final Evaluator<String> timeZone;
- public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format) {
+ public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
this.subject = subject;
this.format = format;
+ this.timeZone = timeZone;
}
@Override
@@ -46,8 +49,18 @@ public class StringToDateEvaluator extends DateEvaluator {
return new DateQueryResult(null);
}
+ final SimpleDateFormat sdf = new SimpleDateFormat(formatValue, Locale.US);
+
+ if(timeZone != null) {
+ final QueryResult<String> tzResult = timeZone.evaluate(attributes);
+ final String tz = tzResult.getValue();
+ if(tz != null && TimeZone.getTimeZone(tz) != null) {
+ sdf.setTimeZone(TimeZone.getTimeZone(tz));
+ }
+ }
+
try {
- return new DateQueryResult(new SimpleDateFormat(formatValue, Locale.US).parse(subjectValue));
+ return new DateQueryResult(sdf.parse(subjectValue));
} catch (final ParseException e) {
throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: "
+ formatValue + "; attribute value: " + subjectValue);
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
index 2dad9ca..03de38f 100644
--- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
+++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
@@ -193,9 +193,8 @@ public class TestQuery {
}
@Test
- @Ignore("Depends on TimeZone")
public void testDateToNumber() {
- final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber()}");
+ final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS', 'America/New_York'):toNumber()}");
final Map<String, String> attributes = new HashMap<>();
attributes.put("dateTime", "2013/11/18 10:22:27.678");
@@ -1332,6 +1331,15 @@ public class TestQuery {
}
@Test
+ public void testDateFormatConversionWithTimeZone() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("blue", "20130917162643");
+ verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'GMT')}", attributes, "2013/09/17 16:26:43.000Z");
+ verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'Europe/Paris')}", attributes, "2013/09/17 18:26:43.000Z");
+ verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'America/Los_Angeles')}", attributes, "2013/09/17 09:26:43.000Z");
+ }
+
+ @Test
public void testNot() {
verifyEquals("${ab:notNull():not()}", new HashMap<String, String>(), true);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/3bb876eb/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
index 2b629e5..7313ce8 100644
--- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
@@ -1754,13 +1754,15 @@ In order to run the correct method, the parameter types must be correct. The Exp
*Description*: [.description]#Formats a number as a date/time according to the format specified by the argument. The argument
must be a String that is a valid Java SimpleDateFormat format. The Subject is expected to be a Number that
- represents the number of milliseconds since Midnight GMT on January 1, 1970.#
+ represents the number of milliseconds since Midnight GMT on January 1, 1970. The number will be evaluated using the local
+ time zone unless specified in the second optional argument.#
*Subject Type*: [.subject]#Number#
*Arguments*:
- [.argName]#_format_# : [.argDesc]#The format to use in the Java SimpleDateFormat syntax#
+ - [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use (in the Java TimeZone syntax)#
*Return Type*: [.returnType]#String#
@@ -1770,10 +1772,12 @@ In order to run the correct method, the parameter types must be correct. The Exp
.format Examples
|============================================================================
| Expression | Value
-| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` | `2014/12/31 15:36:03.264Z`
-| `${time:format("yyyy/MM/dd")}` | `2014/12/31`
-| `${time:format("HH:mm:ss.SSS'Z'")}` | `15:36:03.264Z`
-| `${time:format("2014")}` | `2014`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` | `2014/12/31 20:36:03.264Z`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "America/Los_Angeles")}` | `2014/12/31 12:36:03.264Z`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "Asia/Tokyo")}` | `2015/01/01 05:36:03.264Z`
+| `${time:format("yyyy/MM/dd", "GMT")}` | `2014/12/31`
+| `${time:format("HH:mm:ss.SSS'Z'", "GMT")}` | `20:36:03.264Z`
+| `${time:format("yyyy", "GMT")}` | `2014`
|============================================================================
@@ -1785,20 +1789,21 @@ In order to run the correct method, the parameter types must be correct. The Exp
*Description*: [.description]#Converts a String into a Date data type, based on the format specified by the argument. The argument
must be a String that is a valid Java SimpleDateFormat syntax. The Subject is expected to be a String that is formatted
- according the argument.#
+ according the argument. The date will be evaluated using the local time zone unless specified in the second optional argument.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_format_# : [.argDesc]#The current format to use when parsing the Subject, in the Java SimpleDateFormat syntax.#
+ - [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use when parsing the Subject, in the Java TimeZone syntax.#
*Return Type*: [.returnType]#Date#
*Examples*: If the attribute "year" has the value "2014" and the attribute "time" has the value "2014/12/31 15:36:03.264Z",
- then the Expression `${year:toDate('yyyy')}` will return a Date data type with a value representing Midnight GMT on
- January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` will result in a Date data type for
+ then the Expression `${year:toDate('yyyy', 'GMT')}` will return a Date data type with a value representing Midnight GMT on
+ January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` will result in a Date data type for
15:36:03.264 GMT on December 31, 2014.
Often, this function is used in conjunction with the <<format>> function to change the format of a date/time. For example,