You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2015/04/29 00:58:34 UTC
oozie git commit: OOZIE-2130 Add EL Function for offsetting a date by
a timezone amount including DST (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 9cea0e756 -> 7c78935b3
OOZIE-2130 Add EL Function for offsetting a date by a timezone amount including DST (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7c78935b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7c78935b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7c78935b
Branch: refs/heads/master
Commit: 7c78935b315895ad65b1fdd651825ccde5566e66
Parents: 9cea0e7
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Apr 28 15:58:00 2015 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Tue Apr 28 15:58:00 2015 -0700
----------------------------------------------------------------------
.../apache/oozie/coord/CoordELFunctions.java | 37 ++++++++++---
.../java/org/apache/oozie/util/DateUtils.java | 12 +++++
core/src/main/resources/oozie-default.xml | 4 ++
.../oozie/coord/TestCoordELFunctions.java | 19 +++++--
.../site/twiki/CoordinatorFunctionalSpec.twiki | 56 ++++++++++++++++++--
release-log.txt | 1 +
6 files changed, 115 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
index 7f59186..0cf2c44 100644
--- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
+++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
@@ -167,13 +167,13 @@ public class CoordELFunctions {
}
/**
- * Returns the a date string while given a base date in 'strBaseDate',
- * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH).
+ * Returns a date string that is offset from 'strBaseDate' by the amount specified. The unit can be one of
+ * DAY, MONTH, HOUR, MINUTE, MONTH.
*
- * @param strBaseDate -- base date
- * @param offset -- any number
- * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
- * @return date string
+ * @param strBaseDate The base date
+ * @param offset any number
+ * @param unit one of DAY, MONTH, HOUR, MINUTE, MONTH
+ * @return the offset date string
* @throws Exception
*/
public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
@@ -189,6 +189,27 @@ public class CoordELFunctions {
}
/**
+ * Returns a date string that is offset from 'strBaseDate' by the difference from Oozie processing timezone to the given
+ * timezone. It will account for daylight saving time based on the given 'strBaseDate' and 'timezone'.
+ *
+ * @param strBaseDate The base date
+ * @param timezone
+ * @return the offset date string
+ * @throws Exception
+ */
+ public static String ph2_coord_dateTzOffset(String strBaseDate, String timezone) throws Exception {
+ Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
+ StringBuilder buffer = new StringBuilder();
+ baseCalDate.setTimeZone(DateUtils.getTimeZone(timezone));
+ buffer.append(DateUtils.formatDate(baseCalDate));
+ return buffer.toString();
+ }
+
+ public static String ph3_coord_dateTzOffset(String strBaseDate, String timezone) throws Exception{
+ return ph2_coord_dateTzOffset(strBaseDate, timezone);
+ }
+
+ /**
* Determine the date-time in Oozie processing timezone of n-th future available dataset instance
* from nominal Time but not beyond the instance specified as 'instance.
* <p/>
@@ -770,6 +791,10 @@ public class CoordELFunctions {
return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit);
}
+ public static String ph1_coord_dateTzOffset_echo(String n, String timezone) {
+ return echoUnResolved("dateTzOffset", n + " , " + timezone);
+ }
+
public static String ph1_coord_formatTime_echo(String dateTime, String format) {
// Quote the dateTime value since it would contain a ':'.
return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format);
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/java/org/apache/oozie/util/DateUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/DateUtils.java b/core/src/main/java/org/apache/oozie/util/DateUtils.java
index ec9d0be..2b17a78 100644
--- a/core/src/main/java/org/apache/oozie/util/DateUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/DateUtils.java
@@ -216,6 +216,18 @@ public class DateUtils {
}
/**
+ * Formats a {@link Calendar} as a string in ISO8601 format without adjusting its timezone. However, the mask will still
+ * ensure that the returned date is in the Oozie processing timezone.
+ *
+ * @param c {@link Calendar} to format.
+ * @return the ISO8601 string for the given date, <code>NULL</code> if the {@link Calendar} instance was
+ * <code>NULL</code>
+ */
+ public static String formatDate(Calendar c) {
+ return (c != null) ? getISO8601DateFormat(c.getTimeZone(), ACTIVE_MASK).format(c.getTime()) : "NULL";
+ }
+
+ /**
* This function returns number of hour in a day when given a Calendar with appropriate TZ. It consider DST to find
* the number of hours. Generally it is 24. At some tZ, in one day of a year it is 23 and another day it is 25
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index ecc0d47..1048adc 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -946,6 +946,7 @@
coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap,
coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actualTime_echo_wrap,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo,
@@ -1009,6 +1010,7 @@
coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed,
coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actualTime_echo_wrap,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo,
@@ -1179,6 +1181,7 @@
coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime,
coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actualTime,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name,
@@ -1240,6 +1243,7 @@
coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime,
coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actualTime,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateTzOffset,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime,
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actionId,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph3_coord_name,
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
index 6d3bb48..6515b94 100644
--- a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
+++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
@@ -357,10 +357,8 @@ public class TestCoordELFunctions extends XTestCase {
}
public void testDateOffset() throws Exception {
- init("coord-job-submit-data");
- String expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}";
init("coord-action-start");
- expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}";
+ String expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}";
assertEquals("2009-09-10T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", -1, \"DAY\")}";
@@ -370,6 +368,21 @@ public class TestCoordELFunctions extends XTestCase {
assertEquals("2010-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
}
+ public void testDateTzOffset() throws Exception {
+ init("coord-action-start");
+ // PDT is UTC - 7
+ String expr = "${coord:dateTzOffset(\"2012-06-13T00:00Z\", \"America/Los_Angeles\")}"; //Summer
+ assertEquals("2012-06-12T17:00Z", CoordELFunctions.evalAndWrap(eval, expr));
+ expr = "${coord:dateTzOffset(\"2012-06-13T00:00Z\", \"PST\")}";
+ assertEquals("2012-06-12T17:00Z", CoordELFunctions.evalAndWrap(eval, expr));
+
+ // PST is UTC - 8
+ expr = "${coord:dateTzOffset(\"2012-12-13T00:00Z\", \"America/Los_Angeles\")}"; //Winter
+ assertEquals("2012-12-12T16:00Z", CoordELFunctions.evalAndWrap(eval, expr));
+ expr = "${coord:dateTzOffset(\"2012-12-13T00:00Z\", \"PST\")}";
+ assertEquals("2012-12-12T16:00Z", CoordELFunctions.evalAndWrap(eval, expr));
+ }
+
public void testCurrentRange() throws Exception {
init("coord-action-create");
String expr = "${coord:currentRange(-1, 0)}";
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
index 44a5be2..faff37f 100644
--- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
@@ -3070,16 +3070,22 @@ This section describes the EL functions that could be used to parameterized both
---++++ 6.9.1. coord:dateOffset(String baseDate, int instance, String timeUnit) EL Function
-The =${coord:dateOffset(String baseDate, int instance, String timeUnit)}= EL function calculates date based on the following equation : =newDate = baseDate + instance, * timeUnit=
+The =${coord:dateOffset(String baseDate, int instance, String timeUnit)}= EL function calculates the date based on the following
+equation : =newDate = baseDate + (instance * timeUnit)=
+In other words, it offsets the =baseDate= by the amount specified by =instance= and =timeUnit=.
-For example, if baseDate is '2009-01-01T00:00Z', instance is '2' and timeUnit is 'MONTH', the return date will be '2009-03-01T00:00Z'. If baseDate is '2009-01-01T00:00Z', instance is '1' and timeUnit is 'YEAR', the return date will be '2010-01-01T00:00Z'.
+The =timeUnit= argument accepts one of 'DAY', 'MONTH', 'HOUR', 'MINUTE', 'MONTH'
+
+For example, if =baseDate= is '2009-01-01T00:00Z', =instance= is '2' and =timeUnit= is 'MONTH', the return date will be
+'2009-03-01T00:00Z'. If =baseDate= is '2009-01-01T00:00Z', =instance= is '1' and =timeUnit= is 'YEAR', the return date will be
+'2010-01-01T00:00Z'.
*%GREEN% Example: %ENDCOLOR%*:
<verbatim>
<coordinator-app name="app-coord" frequency="${coord:days(1)}"
- start="2009-01-01T24:00Z" end="2009-12-31T24:00Z" timezone="UTC"
+ start="2009-01-01T23:00Z" end="2009-12-31T23:00Z" timezone="UTC"
xmlns="uri:oozie:coordinator:0.1">
......
<action>
@@ -3100,9 +3106,49 @@ For example, if baseDate is '2009-01-01T00:00Z', instance is '2' and timeUnit is
</coordinator-app>
</verbatim>
-In this example, the 'nextInstance' will be '2009-01-02T24:00Z' for the first action. And the value of 'previousInstance' will be '2008-12-31T24:00Z' for the same instance.
+In this example, the 'nextInstance' will be '2009-01-02T23:00Z' for the first action. And the value of 'previousInstance' will be
+'2008-12-31T23:00Z' for the same instance.
+
+---++++ 6.9.2. coord:dateTzOffset(String baseDate, String timezone) EL Function
+
+The =${coord:dateTzOffset(String baseDate, String timezone)}= EL function calculates the date based on the following
+equation : =newDate = baseDate + (Oozie procesing timezone - timezone)=
+In other words, it offsets the =baseDate= by the difference from Oozie processing timezone to the given =timezone=. It will
+account for daylight saving time based on the given =baseDate= and =timezone=.
+
+The =timezone= argument accepts any timezone or GMT offset that is returned by the
+[[DG_CommandLineTool#Getting_a_list_of_time_zones]["info -timezones"]] command. For example, "America/Los_Angeles" or "PST".
+
+For example, if =baseDate= is '2012-06-13T00:00Z' and =timezone= is 'America/Los_Angeles', the return date will be
+'2012-06-12T17:00Z'. But if =baseDate= is '2012-12-13T00:00Z', then the return date will be '2012-12-12T16:00Z'. The difference
+in return dates occurs because the former occurs during Summer when DST is in effect (UTC-0700) and the latter occurs during Winter
+when DST is no in effect (UTC-0800).
+
+*%GREEN% Example: %ENDCOLOR%*:
+
+
+<verbatim>
+ <coordinator-app name="app-coord" frequency="${coord:days(1)}"
+ start="2009-01-01T24:00Z" end="2009-12-31T24:00Z" timezone="UTC"
+ xmlns="uri:oozie:coordinator:0.1">
+ ......
+ <action>
+ <workflow>
+ <app-path>hdfs://bar:8020/usr/joe/logsaggretor-wf</app-path>
+ <configuration>
+ <property>
+ <name>myDate</name>
+ <value>${coord:dateTzOffset(coord:nominalTime(), "America/Los_Angeles")}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+ </coordinator-app>
+</verbatim>
+
+In this example, the 'myDate' will be '2009-01-01T15:00Z' for the first action.
----++++ 6.9.2. coord:formatTime(String ts, String format) EL Function (since Oozie 2.3.2)
+---++++ 6.9.3. coord:formatTime(String ts, String format) EL Function (since Oozie 2.3.2)
The =${coord:formatTime(String timeStamp, String format)}= function allows transformation of the standard ISO8601 timestamp strings into other desired formats.
http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 48ac8c2..28051b7 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2130 Add EL Function for offsetting a date by a timezone amount including DST (rkanter)
OOZIE-2199 Ooziedb.cmd and oozie-setup.ps1 are missing jars in lib/ for classpath on Windows (venkatnrangan via bzhang)
OOZIE-2012 coordinator with an invalid cron frequency throws NPE after validation (bzhang)
OOZIE-2129 Duplicate child jobs per instance (jaydeepvishwakarma via shwethags)