You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:24 UTC
[19/50] [abbrv] oozie git commit: OOZIE-2630 Oozie Coordinator EL
Functions to get first day of the week/month (satishsaley)
OOZIE-2630 Oozie Coordinator EL Functions to get first day of the week/month (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ff21b990
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ff21b990
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ff21b990
Branch: refs/heads/oya
Commit: ff21b99072203aa7584ddd9a195a32daf97fa52e
Parents: f0c4f2d
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Fri Jan 27 12:28:33 2017 -0800
Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Committed: Fri Jan 27 12:28:33 2017 -0800
----------------------------------------------------------------------
.../oozie/command/coord/CoordCommandUtils.java | 105 +++++--
.../apache/oozie/coord/CoordELFunctions.java | 51 ++++
.../java/org/apache/oozie/coord/TimeUnit.java | 5 +-
.../java/org/apache/oozie/util/DateUtils.java | 21 +-
core/src/main/resources/oozie-default.xml | 10 +
.../command/coord/TestCoordCommandUtils.java | 262 +++++++++++++++-
.../TestCoordMaterializeTransitionXCommand.java | 47 +++
.../oozie/coord/TestCoordELFunctions.java | 8 +
.../test/resources/coord-dataset-endOfDays.xml | 48 +++
.../resources/coord-dataset-endOfMonths.xml | 48 +++
.../test/resources/coord-dataset-endOfWeeks.xml | 48 +++
.../site/twiki/CoordinatorFunctionalSpec.twiki | 303 ++++++++++++++++++-
release-log.txt | 1 +
13 files changed, 913 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
index 0af7edc..63287b9 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
@@ -69,12 +69,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.oozie.CoordinatorJobBean;
public class CoordCommandUtils {
- public static int CURRENT = 0;
- public static int LATEST = 1;
- public static int FUTURE = 2;
- public static int OFFSET = 3;
- public static int ABSOLUTE = 4;
- public static int UNEXPECTED = -1;
+ public static final int CURRENT = 0;
+ public static final int LATEST = 1;
+ public static final int FUTURE = 2;
+ public static final int OFFSET = 3;
+ public static final int ABSOLUTE = 4;
+ public static final int ENDOFMONTHS = 5;
+ public static final int ENDOFWEEKS = 6;
+ public static final int ENDOFDAYS = 7;
+ public static final int UNEXPECTED = -1;
public static final String RESOLVED_UNRESOLVED_SEPARATOR = "!!";
public static final String UNRESOLVED_INSTANCES_TAG = "unresolved-instances";
@@ -90,8 +93,8 @@ public class CoordCommandUtils {
*/
public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception {
int funcType = getFuncType(function);
- if (funcType == ABSOLUTE) {
- return ABSOLUTE;
+ if (funcType == ABSOLUTE || funcType == ENDOFMONTHS || funcType == ENDOFWEEKS || funcType == ENDOFDAYS) {
+ return funcType;
}
if (funcType == CURRENT || funcType == LATEST) {
return parseOneArg(function);
@@ -171,6 +174,15 @@ public class CoordCommandUtils {
else if (function.indexOf("absolute") >= 0) {
return ABSOLUTE;
}
+ else if (function.indexOf("endOfMonths") >= 0) {
+ return ENDOFMONTHS;
+ }
+ else if (function.indexOf("endOfWeeks") >= 0) {
+ return ENDOFWEEKS;
+ }
+ else if (function.indexOf("endOfDays") >= 0) {
+ return ENDOFDAYS;
+ }
return UNEXPECTED;
// throw new RuntimeException("Unexpected instance name "+ function);
}
@@ -182,11 +194,12 @@ public class CoordCommandUtils {
*/
public static void checkIfBothSameType(String startInst, String endInst) throws CommandException {
if (getFuncType(startInst) != getFuncType(endInst)) {
- if (getFuncType(startInst) == ABSOLUTE) {
+ if (getFuncType(startInst) == ABSOLUTE || getFuncType(startInst) == ENDOFMONTHS
+ || getFuncType(startInst) == ENDOFWEEKS || getFuncType(startInst) == ENDOFDAYS) {
if (getFuncType(endInst) != CURRENT) {
throw new CommandException(ErrorCode.E1010,
- "Only start-instance as absolute and end-instance as current is supported." + " start = "
- + startInst + " end = " + endInst);
+ "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays and end-instance as current is supported."
+ + " start = " + startInst + " end = " + endInst);
}
}
else {
@@ -243,6 +256,7 @@ public class CoordCommandUtils {
// arguments for
// future
// function
+
int startIndex = getInstanceNumber(strStart, restArg);
String startRestArg = restArg.toString();
restArg.delete(0, restArg.length());
@@ -251,16 +265,17 @@ public class CoordCommandUtils {
int funcType = getFuncType(strStart);
if (funcType == ABSOLUTE) {
- StringBuffer bf = new StringBuffer();
- bf.append("${coord:absoluteRange(\"").append(parseOneStringArg(strStart))
- .append("\",").append(endIndex).append(")}");
- String matInstance = materializeInstance(event, bf.toString(), appInst, conf, eval);
- if (matInstance != null && !matInstance.isEmpty()) {
- if (instances.length() > 0) {
- instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
- }
- instances.append(matInstance);
- }
+ resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex,
+ parseOneStringArg(strStart));
+ }
+ else if (funcType == ENDOFMONTHS) {
+ resolveInstanceRangeEndOfMonths(event, instances, appInst, conf, eval, strStart, endIndex);
+ }
+ else if (funcType == ENDOFWEEKS) {
+ resolveInstanceRangeEndOfWeeks(event, instances, appInst, conf, eval, strStart, endIndex);
+ }
+ else if (funcType == ENDOFDAYS) {
+ resolveInstanceRangeEndOfDays(event, instances, appInst, conf, eval, strStart, endIndex);
}
else {
if (funcType == OFFSET) {
@@ -326,6 +341,54 @@ public class CoordCommandUtils {
}
}
+ private static void resolveInstanceRangeEndOfDays(Element event, StringBuilder instances, SyncCoordAction appInst,
+ Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception {
+ Date nominalTime = appInst.getNominalTime();
+ Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime));
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ int diff = Integer.parseInt(parseOneStringArg(strStart));
+ cal.add(Calendar.DATE, diff);
+ resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex,
+ DateUtils.formatDateOozieTZ(cal.getTime()));
+ }
+
+ private static void resolveInstanceRangeEndOfWeeks(Element event, StringBuilder instances, SyncCoordAction appInst,
+ Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception {
+ Date nominalTime = appInst.getNominalTime();
+ Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime));
+ cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+ int diff = Integer.parseInt(parseOneStringArg(strStart));
+ cal.add(Calendar.WEEK_OF_YEAR, diff);
+ resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex,
+ DateUtils.formatDateOozieTZ(cal.getTime()));
+ }
+
+ private static void resolveInstanceRangeEndOfMonths(Element event, StringBuilder instances, SyncCoordAction appInst,
+ Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception {
+ int FIRST_DAY_OF_MONTH = 1;
+ Date nominalTime = appInst.getNominalTime();
+ Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime));
+ cal.set(Calendar.DAY_OF_MONTH, FIRST_DAY_OF_MONTH);
+ int diff = Integer.parseInt(parseOneStringArg(strStart));
+ cal.add(Calendar.MONTH, diff);
+ resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex,
+ DateUtils.formatDateOozieTZ(cal.getTime()));
+ }
+
+ private static void resolveAbsoluteRange(Element event, StringBuilder instances, SyncCoordAction appInst,
+ Configuration conf, ELEvaluator eval, String strStart, int endIndex, String rangeStr) throws Exception {
+ StringBuffer bf = new StringBuffer();
+ bf.append("${coord:absoluteRange(\"").append(rangeStr).append("\",")
+ .append(endIndex).append(")}");
+ String matInstance = materializeInstance(event, bf.toString(), appInst, conf, eval);
+ if (matInstance != null && !matInstance.isEmpty()) {
+ if (instances.length() > 0) {
+ instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
+ }
+ instances.append(matInstance);
+ }
+ }
+
/**
* Materialize one instance like current(-2)
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
index 925a7aa..bc18f4d 100644
--- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
+++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
@@ -62,6 +62,8 @@ public class CoordELFunctions {
public static final long MINUTE_MSEC = 60 * 1000L;
public static final long HOUR_MSEC = 60 * MINUTE_MSEC;
public static final long DAY_MSEC = 24 * HOUR_MSEC;
+ public static final long WEEK_MSEC = 7 * DAY_MSEC;
+
/**
* Used in defining the frequency in 'day' unit. <p> domain: <code> val > 0</code> and should be integer.
*
@@ -135,6 +137,27 @@ public class CoordELFunctions {
}
/**
+ * Used in defining the frequency in 'week' unit and specify the "end of
+ * week" property.
+ * <p>
+ * Every instance will start at 00:00 hour of start of week
+ * <p>
+ * domain: <code> val > 0</code> and should be integer.
+ *
+ * @param val frequency in number of weeks.
+ * @return number of weeks and also set the frequency timeunit to week of
+ * the year and end_of_duration flag to week of the year
+ */
+ public static int ph1_coord_endOfWeeks(int val) {
+ val = ParamChecker.checkGTZero(val, "n");
+ ELEvaluator eval = ELEvaluator.getCurrent();
+ eval.setVariable("timeunit", TimeUnit.WEEK);
+ eval.setVariable("endOfDuration", TimeUnit.END_OF_WEEK);
+ return val;
+ }
+
+
+ /**
* Used in defining the frequency in 'month' unit and specify the "end of month" property. <p> Every instance will
* start at first day of each month at 00:00 hour. <p> domain: <code> val > 0</code> and should be integer.
*
@@ -775,6 +798,18 @@ public class CoordELFunctions {
return echoUnResolved("absolute", date);
}
+ public static String ph1_coord_endOfMonths_echo(String date) {
+ return echoUnResolved("endOfMonths", date);
+ }
+
+ public static String ph1_coord_endOfWeeks_echo(String date) {
+ return echoUnResolved("endOfWeeks", date);
+ }
+
+ public static String ph1_coord_endOfDays_echo(String date) {
+ return echoUnResolved("endOfDays", date);
+ }
+
public static String ph1_coord_currentRange_echo(String start, String end) {
return echoUnResolved("currentRange", start + ", " + end);
}
@@ -799,6 +834,18 @@ public class CoordELFunctions {
return echoUnResolved("absolute", date);
}
+ public static String ph2_coord_endOfMonths_echo(String date) {
+ return echoUnResolved("endOfMonths", date);
+ }
+
+ public static String ph2_coord_endOfWeeks_echo(String date) {
+ return echoUnResolved("endOfWeeks", date);
+ }
+
+ public static String ph2_coord_endOfDays_echo(String date) {
+ return echoUnResolved("endOfDays", date);
+ }
+
public static String ph2_coord_absolute_range(String startInstance, int end) throws Exception {
int[] instanceCount = new int[1];
@@ -1397,6 +1444,10 @@ public class CoordELFunctions {
case END_OF_DAY:
instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / DAY_MSEC);
break;
+ case WEEK:
+ case END_OF_WEEK:
+ instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / WEEK_MSEC);
+ break;
case MONTH:
case END_OF_MONTH:
int diffYear = calEffectiveTime.get(Calendar.YEAR) - current.get(Calendar.YEAR);
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/main/java/org/apache/oozie/coord/TimeUnit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/coord/TimeUnit.java b/core/src/main/java/org/apache/oozie/coord/TimeUnit.java
index 5b37639..1a0bf06 100644
--- a/core/src/main/java/org/apache/oozie/coord/TimeUnit.java
+++ b/core/src/main/java/org/apache/oozie/coord/TimeUnit.java
@@ -21,8 +21,9 @@ package org.apache.oozie.coord;
import java.util.Calendar;
public enum TimeUnit {
- MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), MONTH(Calendar.MONTH), YEAR(Calendar.YEAR), END_OF_DAY(Calendar.DATE), END_OF_MONTH(
- Calendar.MONTH), CRON(0), NONE(-1);
+ MINUTE(Calendar.MINUTE), HOUR(Calendar.HOUR), DAY(Calendar.DATE), WEEK(Calendar.WEEK_OF_YEAR), MONTH(
+ Calendar.MONTH), YEAR(Calendar.YEAR), END_OF_DAY(
+ Calendar.DATE), END_OF_MONTH(Calendar.MONTH), END_OF_WEEK(Calendar.WEEK_OF_YEAR), CRON(0), NONE(-1);
private int calendarUnit;
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/main/java/org/apache/oozie/util/DateUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/DateUtils.java b/core/src/main/java/org/apache/oozie/util/DateUtils.java
index 3caf0a2..3c39e7a 100644
--- a/core/src/main/java/org/apache/oozie/util/DateUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/DateUtils.java
@@ -295,14 +295,19 @@ public class DateUtils {
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
}
- else {
- if (endOfFlag == TimeUnit.END_OF_MONTH) {
- cal.add(Calendar.MONTH, 1);
- cal.set(Calendar.DAY_OF_MONTH, 1);
- cal.set(Calendar.HOUR_OF_DAY, 0);
- cal.set(Calendar.MINUTE, 0);
- cal.set(Calendar.SECOND, 0);
- }
+ else if (endOfFlag == TimeUnit.END_OF_MONTH) {
+ cal.add(Calendar.MONTH, 1);
+ cal.set(Calendar.DAY_OF_MONTH, 1);
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 0);
+ }
+ else if (endOfFlag == TimeUnit.END_OF_WEEK) {
+ cal.add(Calendar.WEEK_OF_YEAR, 1);
+ cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 0);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index ad10386..ff9da58 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -804,6 +804,7 @@ will be the requeue interval for the actions which are waiting for a long time w
coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes,
coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfDays,
coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfMonths,
+ coord:endOfWeeks=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfWeeks,
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf
@@ -961,6 +962,9 @@ will be the requeue interval for the actions which are waiting for a long time w
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo,
+ coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfMonths_echo,
+ coord:endOfWeeks=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfWeeks_echo,
+ coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfDays_echo,
hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,
coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo
@@ -1148,6 +1152,9 @@ will be the requeue interval for the actions which are waiting for a long time w
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo,
+ coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfMonths_echo,
+ coord:endOfWeeks=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfWeeks_echo,
+ coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfDays_echo,
coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range,
hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf
</value>
@@ -1207,6 +1214,9 @@ will be the requeue interval for the actions which are waiting for a long time w
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo,
coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range,
+ coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfMonths_echo,
+ coord:endOfWeeks=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfWeeks_echo,
+ coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph2_coord_endOfDays_echo,
hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf,
coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset,
coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java
index 7062e69..22d1f61 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java
@@ -21,8 +21,12 @@ package org.apache.oozie.command.coord;
import java.io.IOException;
import java.io.StringReader;
import java.text.ParseException;
+import java.util.Calendar;
import java.util.Date;
import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.client.CoordinatorJob;
@@ -41,8 +45,6 @@ import org.jdom.JDOMException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
public class TestCoordCommandUtils extends XDataTestCase {
protected Services services;
@@ -455,7 +457,8 @@ public class TestCoordCommandUtils extends XDataTestCase {
}
catch (Exception e) {
assertTrue(e.getMessage().contains(
- "Only start-instance as absolute and end-instance as current is supported"));
+ "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays"
+ + " and end-instance as current is supported."));
}
}
@@ -514,6 +517,229 @@ public class TestCoordCommandUtils extends XDataTestCase {
assertNull(CoordCommandUtils.computeNextNominalTime(job, action));
}
+ @Test
+ public void testCoordEndOfMonthsParamerized() throws Exception {
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(0)",
+ "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ Element e1 = (Element) elementList.get(0);
+ Element e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfMonths(0) i.e.2009-08-01T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ Calendar start = DateUtils.getCalendar("2009-08-01T01:00Z", DateUtils.UTC);
+ checkUris(e1.getChild("uris", e1.getNamespace()).getTextTrim(), (Calendar) start.clone(), Calendar.DATE, 1,
+ "YYYY/MM/dd");
+
+ // Test parameterized
+ // startInstance = coord:endOfMonths(0) i.e.2009-08-01T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd");
+ }
+
+ public void testCoordEndOfMonthsStartingFromPrevMonth() throws Exception {
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-1)",
+ "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ Element e2 = (Element) elementList.get(1);
+ e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfMonths(0) i.e.2009-07-01T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ Calendar start = DateUtils.getCalendar("2009-07-01T01:00Z", DateUtils.UTC);
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd");
+ }
+
+ public void testCoordEndOfMonthsFailOnStartInstanceIsLater() throws Exception {
+ try {
+ // start instance = 2009-10-01T01:00Z
+ // end instance = 2009-08-01T01:00Z
+ getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(2)",
+ "coord:current(0)", "2009-08-06T01:00Z", "2009-08-16T01:00Z");
+ fail("Should throw exception because end is earlier than start-instance");
+ }
+ catch (Exception e) {
+ assertTrue(e.getCause().getMessage()
+ .contains("start-instance should be equal or earlier than the end-instance"));
+ }
+ }
+
+ public void testCoordEndOfMonthsFailOnInitialInstanceIsLater() throws Exception {
+ try {
+ // start instance = 2009-06-01T01:00Z
+ // initial instance = 2009-08-06T01:00Z
+ getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-2)", "coord:current(0)",
+ "2009-08-06T01:00Z", "2009-08-16T01:00Z");
+ fail("Should throw exception because initial instance is later than start instance");
+ }
+ catch (Exception e) {
+ e.printStackTrace(System.out);
+ assertTrue(e.getCause().getMessage()
+ .contains("intial-instance should be equal or earlier than the start-instance"));
+ }
+ }
+
+ public void testCoordEndOfMonthsFailOnLatestAsEndInstance() throws Exception {
+ try {
+ getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(0)",
+ "coord:latest(2)", "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ fail("Should throw exception because latest is not allowed as end-instance");
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage().contains(
+ "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays"
+ + " and end-instance as current is supported."));
+ }
+ }
+
+ @Test
+ public void testCoordEndOfWeeksParamerized() throws Exception {
+ String nominalTime = "2009-08-20T01:00Z";
+ String startTime = nominalTime;
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(0)",
+ "coord:current(0)", nominalTime, startTime);
+ Element e1 = (Element) elementList.get(0);
+ Element e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfWeeks(0)
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ Calendar start = DateUtils.getCalendar(nominalTime, DateUtils.UTC);
+ start.add(Calendar.DAY_OF_WEEK, start.getFirstDayOfWeek() - start.get(Calendar.DAY_OF_WEEK));
+ checkUris(e1.getChild("uris", e1.getNamespace()).getTextTrim(), (Calendar) start.clone(), Calendar.DATE, 1,
+ "YYYY/MM/dd");
+
+ // Test parameterized
+ // startInstance = coord:endOfWeeks(0)
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd");
+ }
+
+ public void testCoordEndOfWeeksStartingFromPrevWeek() throws Exception {
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-1)",
+ "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ Element e2 = (Element) elementList.get(1);
+ e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfWeeks(-1)
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ Calendar start = DateUtils.getCalendar("2009-08-10T01:00Z", DateUtils.UTC);
+ start.add(Calendar.DAY_OF_WEEK, start.getFirstDayOfWeek() - start.get(Calendar.DAY_OF_WEEK));
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd");
+ }
+
+ public void testCoordEndOfWeeksFailOnStartInstanceIsLater() throws Exception {
+ try {
+ // start instance = coord:endOfWeeks(2) i.e. 2009-08-30T01:00Z
+ // end instance = coord:current(0) i.e. 2009-08-20T01:00Z
+ getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(2)", "coord:current(0)",
+ "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ fail("Should throw exception because end is earlier than start-instance");
+ }
+ catch (Exception e) {
+ assertTrue(e.getCause().getMessage()
+ .contains("start-instance should be equal or earlier than the end-instance"));
+ }
+ }
+
+ public void testCoordEndOfWeeksFailOnInitialInstanceIsLater() throws Exception {
+ try {
+ // start instance = coord:endOfWeeks(-10) i.e. 2009-05-31T01:00Z
+ // initial instance = 2009-06-06T01:00Z
+ getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-10)", "coord:current(0)",
+ "2009-08-06T01:00Z", "2009-08-16T01:00Z");
+ fail("Should throw exception because initial instance is later than start instance");
+ }
+ catch (Exception e) {
+ e.printStackTrace(System.out);
+ assertTrue(e.getCause().getMessage()
+ .contains("intial-instance should be equal or earlier than the start-instance"));
+ }
+ }
+
+ public void testCoordEndOfWeeksFailOnLatestAsEndInstance() throws Exception {
+ try {
+ getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(0)", "coord:latest(2)",
+ "2009-08-20T01:00Z", "2009-08-20T01:00Z");
+ fail("Should throw exception because latest is not allowed as end-instance");
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage().contains(
+ "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays"
+ + " and end-instance as current is supported."));
+ }
+ }
+
+ @Test
+ public void testCoordEndOfDaysParameterized() throws Exception {
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(0)",
+ "coord:current(0)", "2009-08-20T18:00Z", "2009-08-20T18:00Z");
+ Element e1 = (Element) elementList.get(0);
+ Element e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfDays(0) i.e 2009-08-20T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T18:00Z
+ Calendar start = DateUtils.getCalendar("2009-08-20T00:00Z", DateUtils.UTC);
+ checkUris(e1.getChild("uris", e1.getNamespace()).getTextTrim(), (Calendar) start.clone(), Calendar.MINUTE, 30,
+ "YYYY/MM/dd/HH/mm");
+
+ // Test parameterized
+ // startInstance = coord:endOfDays(0) i.e 2009-08-20T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.MINUTE, 30,
+ "YYYY/MM/dd/HH/mm");
+ }
+
+ public void testCoordEndOfDaysStartingFromPrevDay() throws Exception {
+ List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-1)",
+ "coord:current(0)", "2009-08-20T18:00Z", "2009-08-20T18:00Z");
+ Element e2 = (Element) elementList.get(1);
+
+ // startInstance = coord:endOfDays(-1) i.e 2009-08-19T01:00Z and
+ // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z
+ Calendar start = DateUtils.getCalendar("2009-08-19T00:00Z", DateUtils.UTC);
+ checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.MINUTE, 30,
+ "YYYY/MM/dd/HH/mm");
+ }
+
+ public void testCoordEndOfDaysFailOnStartInstanceIsLater() throws Exception {
+ try {
+ // start instance = coord:endOfDays(2) i.e. 2009-08-08T00:00Z
+ // end instance = coord:current(0) i.e. 2009-08-06T18:00Z
+ getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(2)",
+ "coord:current(0)", "2009-08-06T18:00Z", "2009-08-16T18:00Z");
+ }
+ catch (Exception e) {
+ assertTrue(e.getCause().getMessage()
+ .contains("start-instance should be equal or earlier than the end-instance"));
+ }
+ }
+
+ public void testCoordEndOfDaysFailOnInitialInstanceIsLater() throws Exception {
+ try {
+ // start instance = coord:endOfDays(-10) i.e. 2009-07-27T00:00Z
+ // initial instance = 2009-08-01T01:00Z
+ getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-10)", "coord:current(0)",
+ "2009-08-06T18:00Z", "2009-08-16T18:00Z");
+ fail("Should throw exception because initial instance is later than start instance");
+ }
+ catch (Exception e) {
+ assertTrue(e.getCause().getMessage()
+ .contains("intial-instance should be equal or earlier than the start-instance"));
+ }
+ }
+
+ public void testCoordEndOfDaysFailOnLatestAsEndInstance() throws Exception {
+ try {
+ getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(0)",
+ "coord:latest(2)", "2009-08-20T18:00Z", "2009-08-20T18:00Z");
+ fail("Should throw exception. Start-instance is coord:endOfDays and end-instance is latest");
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage()
+ .contains(
+ "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays"
+ + " and end-instance as current is supported."));
+ }
+ }
+
protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime,
String freq) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, startTime, endTime, false, false, 0);
@@ -598,4 +824,34 @@ public class TestCoordCommandUtils extends XDataTestCase {
addPartition(db, table, "dt=20120413;country=brazil");
}
+ private void checkUris(String uris, Calendar start, int freqUnit ,int freq, String format) {
+ //String format = "YYYY/MM/dd";
+ String[] allUris = uris.split("#");
+ String pathFormat = "hdfs:///tmp/workflows/%s;region=us";
+ for (int i = allUris.length - 1; i >= 0; i--) {
+ String path = String.format(pathFormat, DateUtils.formatDateCustom(start.getTime(), format));
+ assertEquals(allUris[i], path);
+ start.add(freqUnit, freq);
+ }
+ }
+
+ private List<?> getSyncDatasetEvents(String coordXml, String startInstanceParam, String endInstanceParam,
+ String nominalTime, String actualTime) throws Exception {
+ CoordinatorJobBean job = addRecordToCoordJobTableForWaiting(coordXml, CoordinatorJob.Status.RUNNING, false,
+ true);
+ Path appPath = new Path(getFsTestCaseDir(), "coord");
+ String actionXml = getCoordActionXml(appPath, coordXml);
+ CoordinatorActionBean actionBean = createCoordinatorActionBean(job);
+ Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
+ Element eAction = createActionElement(actionXml);
+ jobConf.set("startInstance", startInstanceParam);
+ jobConf.set("endInstance", endInstanceParam);
+ String output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction,
+ DateUtils.parseDateOozieTZ(nominalTime), DateUtils.parseDateOozieTZ(actualTime), 1, jobConf,
+ actionBean);
+ eAction = XmlUtils.parseXml(output);
+ List<?> elementList = ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
+ .getChildren();
+ return elementList;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
index 29e7ca1..2e39689 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
@@ -743,6 +743,53 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
assertEquals(job.getNextMaterializedTime(), origStart.getTime());
}
+ public void testActionMaterEndOfWeeks() throws Exception {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ String appXml = "<coordinator-app name=\"test\" frequency=\"${coord:endOfWeeks(1)}\" start=\"2016-02-03T01:00Z\" "
+ + "end=\"2016-03-03T23:59Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> "
+ + "<execution>LIFO</execution> </controls> <datasets> "
+ + "<dataset name=\"a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> " + "</dataset> "
+ + "<dataset name=\"local_a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> " + " </dataset> " + "</datasets> <input-events> "
+ + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in> "
+ + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> "
+ + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> "
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> "
+ + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> "
+ + "</property></configuration> </workflow> </action> </coordinator-app>";
+ writeToFile(appXml, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+
+ CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
+ assertEquals(job.getLastActionNumber(), 0);
+
+ job.setMatThrottling(10);
+ CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
+ assertEquals(4, job.getLastActionNumber());
+
+ String jobXml = job.getJobXml();
+ Element eJob = XmlUtils.parseXml(jobXml);
+ TimeZone appTz = DateUtils.getTimeZone(job.getTimeZone());
+ TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
+ TimeUnit freqTU = TimeUnit.valueOf(job.getTimeUnitStr());
+ Calendar origStart = Calendar.getInstance(appTz);
+ origStart.setTime(job.getStartTimestamp());
+ // Move to the End of duration, if needed.
+ DateUtils.moveToEnd(origStart, endOfFlag);
+ origStart.add(freqTU.getCalendarUnit(), 4 * Integer.parseInt(job.getFrequency()));
+ assertEquals(origStart.getTime(), job.getNextMaterializedTime());
+ }
+
protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime,
Date pauseTime, String freq) throws Exception {
return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq);
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
index be60133..7aa91e8 100644
--- a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
+++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
@@ -121,6 +121,14 @@ public class TestCoordELFunctions extends XTestCase {
assertEquals(TimeUnit.END_OF_DAY, (TimeUnit) eval.getVariable("endOfDuration"));
}
+ public void testEndOfWeeks() throws Exception {
+ init("coord-job-submit-freq");
+ String expr = "${coord:endOfWeeks(3)}";
+ assertEquals("3", CoordELFunctions.evalAndWrap(eval, expr));
+ assertEquals(TimeUnit.WEEK, (TimeUnit) eval.getVariable("timeunit"));
+ assertEquals(TimeUnit.END_OF_WEEK, (TimeUnit) eval.getVariable("endOfDuration"));
+ }
+
public void testEndOfMonths() throws Exception {
init("coord-job-submit-freq");
String expr = "${coord:endOfMonths(1)}";
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/resources/coord-dataset-endOfDays.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-dataset-endOfDays.xml b/core/src/test/resources/coord-dataset-endOfDays.xml
new file mode 100644
index 0000000..6aa3f00
--- /dev/null
+++ b/core/src/test/resources/coord-dataset-endOfDays.xml
@@ -0,0 +1,48 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<coordinator-app xmlns='uri:oozie:coordinator:0.2' name='NAME'
+ frequency="1" start='2009-08-20T01:00Z' end='2009-10-10T23:59Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <controls>
+ <timeout>10</timeout>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='logs' frequency='30' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='MINUTE' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfDays(0)}
+ </start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ <data-in name="input" dataset="test">
+ <dataset name='test' frequency='30' initial-instance='2009-08-01T01:00Z'
+ timezone='UTC' freq_timeunit='MINUTE' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${startInstance}</start-instance>
+ <end-instance>${endInstance}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/resources/coord-dataset-endOfMonths.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-dataset-endOfMonths.xml b/core/src/test/resources/coord-dataset-endOfMonths.xml
new file mode 100644
index 0000000..0ea5cac
--- /dev/null
+++ b/core/src/test/resources/coord-dataset-endOfMonths.xml
@@ -0,0 +1,48 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<coordinator-app xmlns='uri:oozie:coordinator:0.2' name='NAME'
+ frequency="1" start='2009-08-20T01:00Z' end='2009-10-10T23:59Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <controls>
+ <timeout>10</timeout>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='logs' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfMonths(0)}
+ </start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ <data-in name="input" dataset="test">
+ <dataset name='test' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${startInstance}</start-instance>
+ <end-instance>${endInstance}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/core/src/test/resources/coord-dataset-endOfWeeks.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-dataset-endOfWeeks.xml b/core/src/test/resources/coord-dataset-endOfWeeks.xml
new file mode 100644
index 0000000..7879bf4
--- /dev/null
+++ b/core/src/test/resources/coord-dataset-endOfWeeks.xml
@@ -0,0 +1,48 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<coordinator-app xmlns='uri:oozie:coordinator:0.2' name='NAME'
+ frequency="1" start='2009-08-20T01:00Z' end='2009-10-10T23:59Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <controls>
+ <timeout>10</timeout>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='logs' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfWeeks(0)}
+ </start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ <data-in name="input" dataset="test">
+ <dataset name='test' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us
+ </uri-template>
+ </dataset>
+ <start-instance>${startInstance}</start-instance>
+ <end-instance>${endInstance}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
index e3ac514..744f5d9 100644
--- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
@@ -320,7 +320,7 @@ The =${coord:months(int n)}= and =${coord:endOfMonths(int n)}= EL functions shou
Constant values cannot be used to indicate a month based frequency because the number of days in a month changes month to month and on leap years; plus the number of hours in every day of the month are not always the same for timezones that observe daylight-saving time.
----++++ 4.4.2.1. The coord:months(int n) EL function
+---+++++ 4.4.2.1. The coord:months(int n) EL function
The =${coord:months(int n)}= EL function returns the number of minutes for 'n' complete months starting with the month of the current nominal time for which the computation is being done.
@@ -339,7 +339,7 @@ The =${coord:months(int n)}= EL function includes *all* the minutes of the curre
| =2009-03-08T08:00Z= | =UTC= | =${coord:months(2)}= | 87840 | =2009-03-08T08:00Z= | total minutes in 2009MAR and 2009APR UTC time |
| =2009-03-08T08:00Z= | =America/Los_Angeles= | =${coord:months(2)}= | 87780 | =2009-03-08T08:00Z= | total minutes in 2009MAR and 2009APR PST8PDT time <br/> (2009MAR08 is DST switch in US) |
----++++ 4.4.2.2. The coord:endOfMonths(int n) EL function
+---+++++ 4.4.2.2. The coord:endOfMonths(int n) EL function
The =${coord:endOfMonths(int n)}= EL function is identical to the =${coord:months(int n)}= except that it shifts the first occurrence to the end of the month for the specified timezone before computing the interval in minutes.
@@ -405,7 +405,75 @@ The =${coord:endOfMonths(int n)}= EL function is identical to the =${coord:month
</coordinator-app>
</verbatim>
----++++ 4.4.3. Cron syntax in coordinator frequency
+---++++ 4.4.3. The coord:endOfWeeks(int n) EL function
+
+The =${coord:endOfWeeks(int n)}= EL function shifts the first occurrence to the start of the week for the specified
+timezone before computing the interval in minutes. The start of the week depends on the Java's implementation of
+[[https://docs.oracle.com/javase/8/docs/api/java/util/Calendar.html#getFirstDayOfWeek--][Calendar.getFirstDayOfWeek()]]
+ i.e. first day of the week is SUNDAY in the U.S., MONDAY in France.
+
+*%GREEN% Examples: %ENDCOLOR%*
+
+| *Starting Nominal UTC time* | *Timezone* | *Usage* | *Value* | *First Occurrence* | *Comments* |
+| =2017-01-04T00:00Z= | =UTC= | =${coord:endOfWeeks(1)}= | 10080 | =2017-01-08T00:00Z= | first occurrence on 2017JAN08 08:00 UTC time |
+| =2017-01-04T08:00Z= | =UTC= | =${coord:endOfWeeks(1)}= | 10080 | =2017-01-08T08:00Z= | first occurrence on 2017JAN08 08:00 UTC time |
+| =2017-01-06T08:00Z= | =UTC= | =${coord:endOfWeeks(1)}= | 10080 | =2017-01-08T08:00Z= | first occurrence on 2017JAN08 08:00 UTC time |
+| =2017-01-04T08:00Z= | =America/Los_Angeles= | =${coord:endOfWeeks(1)}= | 10080 | =2017-01-08T08:00Z= | first occurrence in 2017JAN08 08:00 UTC time |
+| =2017-01-06T08:00Z= | =America/Los_Angeles= | =${coord:endOfWeeks(1)}= | 10080 | =2017-01-08T08:00Z= | first occurrence in 2017JAN08 08:00 UTC time |
+
+<verbatim>
+<coordinator-app name="hello-coord" frequency="${coord:endOfWeeks(1)}"
+ start="2017-01-04T08:00Z" end="2017-12-31T08:00Z" timezone="America/Los_Angeles"
+ xmlns="uri:oozie:coordinator:0.1">
+ <controls>
+ <timeout>10</timeout>
+ <concurrency>${concurrency_level}</concurrency>
+ <execution>${execution_order}</execution>
+ <throttle>${materialization_throttle}</throttle>
+ </controls>
+
+ <datasets>
+ <dataset name="din" frequency="${coord:endOfWeeks(1)}"
+ initial-instance="2017-01-01T08:00Z" timezone="America/Los_Angeles">
+ <uri-template>${baseFsURI}/${YEAR}/${MONTH}/${DAY}</uri-template>
+ </dataset>
+ <dataset name="dout" frequency="${coord:endOfWeeks(1)}"
+ initial-instance="2017-01-01T08:00Z" timezone="UTC">
+ <uri-template>${baseFsURI}/${YEAR}/${MONTH}/${DAY}</uri-template>
+ </dataset>
+ </datasets>
+
+ <input-events>
+ <data-in name="input" dataset="din">
+ <instance>${coord:current(0)}</instance>
+ </data-in>
+ </input-events>
+
+ <output-events>
+ <data-out name="output" dataset="dout">
+ <instance>${coord:current(1)}</instance>
+ </data-out>
+ </output-events>
+
+ <action>
+ <workflow>
+ <app-path>${wf_app_path}</app-path>
+ <configuration>
+ <property>
+ <name>wfInput</name>
+ <value>${coord:dataIn('input')}</value>
+ </property>
+ <property>
+ <name>wfOutput</name>
+ <value>${coord:dataOut('output')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+ </coordinator-app>
+</verbatim>
+
+---++++ 4.4.4. Cron syntax in coordinator frequency
Oozie has historically allowed only very basic forms of scheduling: You could choose
to run jobs separated by a certain number of minutes, hours, days or weeks. That's
@@ -2151,7 +2219,7 @@ Then, the dataset instances for the input events for the coordinator action will
---++++ 6.6.8. coord:absolute(String timeStamp) EL Function for Synchronous Datasets
-=${coord:absolute(String timeStamp)} = represents absolute dataset instance time. coord:absolute is only supported with range
+=${coord:absolute(String timeStamp)}= represents absolute dataset instance time. coord:absolute is only supported with range
where, start-instance is coord:absolute and end-instance is coord:current. Specifying a fixed date as the start instance is
useful if your processing needs to process all dataset instances from a specific instance to the current instance.
@@ -2162,13 +2230,13 @@ Coordinator application definition:
<verbatim>
<coordinator-app name="app-coord" frequency="${coord:months(1)}"
- start="2009-01-01T01:00Z" end="2009-12-31T24:00" timezone="UTC"
+ start="2009-01-01T01:00Z" end="2009-12-31T00:00" timezone="UTC"
xmlns="uri:oozie:coordinator:0.4">
<input-events>
<data-in name="input" dataset="logs">
<dataset name='a' frequency='7' initial-instance="2009-01-01T01:00Z"
timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
- <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}/${HOUR}
+ <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}
</uri-template>
</dataset>
<start-instance>${coord:absolute("2009-01-01T01:00Z")}</start-instance>
@@ -2187,23 +2255,238 @@ Then, the dataset instances for the input events for the coordinator action at f
<verbatim>
hdfs://bar:8020/app/logs/2009/02/01
-<verbatim>
+</verbatim>
The dataset instances for the input events for the coordinator action at second run will be:
<verbatim>
hdfs://bar:8020/app/logs/2009/02/01
hdfs://bar:8020/app/logs/2009/02/07
+</verbatim>
+
+
+---++++ 6.6.9. coord:endOfMonths(int n) EL Function for Synchronous Datasets
+
+
+=${coord:endOfMonths(int n)}= represents dataset instance at start of n <sup>th</sup> month. coord:endOfMonths is only
+supported with range, where start-instance is coord:endOfMonths and end-instance is coord:current. Specifying start of
+a month is useful if you want to process all the dataset instances from starting of a month to the current instance.
+This function calculates the dataset occurences similar to
+[[CoordinatorFunctionalSpec#a4.4.2.2._The_coord:endOfMonthsint_n_EL_function][coord:endOfMonths(int n) EL function]]
+mentioned above.
+
+
+*%GREEN% Example: %ENDCOLOR%*:
+
+Coordinator application definition:
+
<verbatim>
+<coordinator-app name="app-coord" frequency="${coord:days(1)}"
+ start='2009-08-06T01:00Z' end="2009-12-31T00:00" timezone="UTC"
+ xmlns="uri:oozie:coordinator:0.4">
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='a' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfMonths(0)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ .............
+ </workflow>
+ </action>
+</coordinator-app>
+</verbatim>
+
+Then, the dataset instances for the input events for the coordinator action at first run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/01
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/03
+ hdfs://bar:8020/app/logs/2009/08/04
+ hdfs://bar:8020/app/logs/2009/08/05
+ hdfs://bar:8020/app/logs/2009/08/06
+</verbatim>
+
+The dataset instances for the input events for the coordinator action at second run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/01
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/03
+ hdfs://bar:8020/app/logs/2009/08/04
+ hdfs://bar:8020/app/logs/2009/08/05
+ hdfs://bar:8020/app/logs/2009/08/06
+ hdfs://bar:8020/app/logs/2009/08/07
+</verbatim>
+
+
+---++++ 6.6.10. coord:endOfWeeks(int n) EL Function for Synchronous Datasets
+
+=${coord:endOfWeeks(int n)}= represents dataset instance at start of n <sup>th</sup> week. The start of the week
+calculated similar to mentioned in
+[[CoordinatorFunctionalSpec#a4.4.3._The_coord:endOfWeeksint_n_EL_function][coord:endOfWeeks section above]].
+coord:endOfWeeks is only supported with range, where start-instance is coord:endOfWeeks and end-instance is
+coord:current. Specifying start of a week is useful if you want to process all the dataset instances from
+starting of a week to the current instance.
----++++ 6.6.9. coord:version(int n) EL Function for Asynchronous Datasets
+*%GREEN% Examples: %ENDCOLOR%*
+
+1. *. =${coord:endOfWeeks(int n)}= datetime calculation:*
+
+Datasets Definition:
+
+<verbatim>
+<datasets>
+.
+ <dataset name="logs" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T00:00Z" timezone="UTC">
+ <uri-template>hdfs://bar:8020/app/logs/${YEAR}${MONTH}/${DAY}</uri-template>
+ </dataset>
+.
+</datasets>
+</verbatim>
+
+For a coordinator action creation time: =2009-05-29T00:00Z= the =${coord:endOfMonths(int n)}= EL function
+would resolve to the following datetime values for the 'logs' dataset:
+
+| *${coord:endOfWeeks(int offset)}* | *Dataset 'logs'* |
+| =${coord:endOfWeeks(0)}= | =2009-05-24T00:00Z= |
+| =${coord:endOfWeeks(1)}= | =2009-05-31T00:00Z= |
+| =${coord:endOfWeeks(-1)}= | =2009-05-17T00:00Z= |
+| =${coord:endOfWeeks(-3)}= | =2009-05-03T00:00Z= |
+
+
+*%GREEN% Example: %ENDCOLOR%*:
+
+Coordinator application definition:
+
+<verbatim>
+<coordinator-app name="app-coord" frequency="${coord:days(1)}"
+ start='2009-08-06T01:00Z' end="2009-12-31T00:00" timezone="UTC"
+ xmlns="uri:oozie:coordinator:0.4">
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='a' frequency='1' initial-instance='2009-06-06T01:00Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfWeeks(0)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ .............
+ </workflow>
+ </action>
+</coordinator-app>
+</verbatim>
+
+Then, the dataset instances for the input events for the coordinator action at first run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/03
+ hdfs://bar:8020/app/logs/2009/08/04
+ hdfs://bar:8020/app/logs/2009/08/05
+ hdfs://bar:8020/app/logs/2009/08/06
+</verbatim>
+
+The dataset instances for the input events for the coordinator action at second run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/02
+ hdfs://bar:8020/app/logs/2009/08/03
+ hdfs://bar:8020/app/logs/2009/08/04
+ hdfs://bar:8020/app/logs/2009/08/05
+ hdfs://bar:8020/app/logs/2009/08/06
+ hdfs://bar:8020/app/logs/2009/08/07
+</verbatim>
+
+
+---++++ 6.6.11. coord:endOfDays(int n) EL Function for Synchronous Datasets
+
+
+=${coord:endOfDays(int n)}= represents dataset instance at start of n <sup>th</sup> day. coord:endOfDays is only
+supported with range, where start-instance is coord:endOfDays and end-instance is coord:current. Specifying start
+of a day is useful if you want to process all the dataset instances from starting of a day to the current instance.
+This function calculates the dataset occurences similar to
+[[CoordinatorFunctionalSpec#a4.4.1.2._The_coord:endOfDaysint_n_EL_function][coord:endOfDays(int n) EL function]]
+mentioned above.
+
+
+*%GREEN% Example: %ENDCOLOR%*:
+
+Coordinator application definition:
+
+<verbatim>
+<coordinator-app name="app-coord" frequency='60' freq_timeunit='MINUTE'
+ start='2009-08-06T05:00Z' end="2009-12-31T00:00" timezone="UTC"
+ xmlns="uri:oozie:coordinator:0.4">
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <dataset name='a' frequency='60' initial-instance='2009-06-06T00:00Z'
+ timezone='UTC' freq_timeunit='MINUTE' end_of_duration='NONE'>
+ <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}
+ </uri-template>
+ </dataset>
+ <start-instance>${coord:endOfDays(0)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ .............
+ </workflow>
+ </action>
+</coordinator-app>
+</verbatim>
+
+Then, the dataset instances for the input events for the coordinator action at first run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/06/00/00
+ hdfs://bar:8020/app/logs/2009/08/06/01/00
+ hdfs://bar:8020/app/logs/2009/08/06/02/00
+ hdfs://bar:8020/app/logs/2009/08/06/03/00
+ hdfs://bar:8020/app/logs/2009/08/06/04/00
+ hdfs://bar:8020/app/logs/2009/08/06/05/00
+</verbatim>
+
+The dataset instances for the input events for the coordinator action at second run will be:
+
+<verbatim>
+ hdfs://bar:8020/app/logs/2009/08/06/00/00
+ hdfs://bar:8020/app/logs/2009/08/06/01/00
+ hdfs://bar:8020/app/logs/2009/08/06/02/00
+ hdfs://bar:8020/app/logs/2009/08/06/03/00
+ hdfs://bar:8020/app/logs/2009/08/06/04/00
+ hdfs://bar:8020/app/logs/2009/08/06/05/00
+ hdfs://bar:8020/app/logs/2009/08/06/06/00
+</verbatim>
+
+
+---++++ 6.6.12. coord:version(int n) EL Function for Asynchronous Datasets
* TBD
----++++ 6.6.10. coord:latest(int n) EL Function for Asynchronous Datasets
+
+---++++ 6.6.13. coord:latest(int n) EL Function for Asynchronous Datasets
* TBD
----++++ 6.6.11. Dataset Instance Resolution for Instances Before the Initial Instance
+
+---++++ 6.6.14. Dataset Instance Resolution for Instances Before the Initial Instance
+
When defining input events that refer to dataset instances it may be possible that the resolution of instances is out of it lower bound. This is scenario is likely to happen when the instance resolution is very close to the initial-instance. This is useful for bootstrapping the application.
http://git-wip-us.apache.org/repos/asf/oozie/blob/ff21b990/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 929b01a..cd40ea6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2630 Oozie Coordinator EL Functions to get first day of the week/month (satishsaley)
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
OOZIE-2619 Make Hive action defaults to match hive defaults when running from command line (venkatnrangan via abhishekbafna)
OOZIE-2776 Log message is repeated twice for hive actions (satishsaley)