You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 19:02:40 UTC
svn commit: r1442249 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/command/coord/
core/src/main/java/org/apache/oozie/coord/ core/src/main/resources/
core/src/test/java/org/apache/oozie/command/coord/
Author: rkanter
Date: Mon Feb 4 18:02:40 2013
New Revision: 1442249
URL: http://svn.apache.org/viewvc?rev=1442249&view=rev
Log:
OOZIE-1073 Optimize latest and future EL resolution in case of start-instance and end-instance (rohini via virag)
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
oozie/branches/branch-3.3/core/src/main/resources/oozie-default.xml
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java?rev=1442249&r1=1442248&r2=1442249&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java Mon Feb 4 18:02:40 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -231,16 +231,13 @@ public class CoordCommandUtils {
}
}
else { // latest(n)/future() EL is present
- for (; startIndex <= endIndex; startIndex++) {
- if (instances.length() > 0) {
- instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
- }
- if (funcType == LATEST) {
- instances.append("${coord:latest(").append(startIndex).append(")}");
- }
- else if (funcType == FUTURE) {
- instances.append("${coord:future(").append(startIndex).append(",'").append(endRestArg).append("')}");
- }
+ if (funcType == LATEST) {
+ instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex)
+ .append(")}");
+ }
+ else if (funcType == FUTURE) {
+ instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex)
+ .append(",'").append(endRestArg).append("')}");
}
}
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java?rev=1442249&r1=1442248&r2=1442249&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java Mon Feb 4 18:02:40 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -41,6 +41,7 @@ import org.apache.oozie.service.HadoopAc
*/
public class CoordELFunctions {
+ final private static XLog LOG = XLog.getLog(CoordELFunctions.class);
final private static String DATASET = "oozie.coord.el.dataset.bean";
final private static String COORD_ACTION = "oozie.coord.el.app.bean";
final public static String CONFIGURATION = "oozie.coord.el.conf";
@@ -221,13 +222,64 @@ public class CoordELFunctions {
}
}
+ /**
+ * Determine the date-time in Oozie processing timezone of the future available dataset instances
+ * from start to end offsets from nominal Time but not beyond the instance specified as 'instance'.
+ * <p/>
+ * It depends on:
+ * <p/>
+ * 1. Data set frequency
+ * <p/>
+ * 2. Data set Time unit (day, month, minute)
+ * <p/>
+ * 3. Data set Time zone/DST
+ * <p/>
+ * 4. End Day/Month flag
+ * <p/>
+ * 5. Data set initial instance
+ * <p/>
+ * 6. Action Creation Time
+ * <p/>
+ * 7. Existence of dataset's directory
+ *
+ * @param start : start instance offset
+ * <p/>
+ * domain: start >= 0, start is integer
+ * @param end : end instance offset
+ * <p/>
+ * domain: end >= 0, end is integer
+ * @param instance: How many future instance it should check? value should
+ * be >=0
+ * @return date-time in Oozie processing timezone of the instances from start to end offsets
+ * delimited by comma.
+ * <p/>
+ * @throws Exception
+ */
+ public static String ph3_coord_futureRange(int start, int end, int instance) throws Exception {
+ ParamChecker.checkGEZero(start, "future:n");
+ ParamChecker.checkGEZero(end, "future:n");
+ ParamChecker.checkGTZero(instance, "future:instance");
+ if (isSyncDataSet()) {// For Sync Dataset
+ return coord_futureRange_sync(start, end, instance);
+ }
+ else {
+ throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
+ }
+ }
+
private static String coord_future_sync(int n, int instance) throws Exception {
+ return coord_futureRange_sync(n, n, instance);
+ }
+
+ private static String coord_futureRange_sync(int startOffset, int endOffset, int instance) throws Exception {
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
int datasetFrequency = (int) getDSFrequency();// in minutes
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
+ StringBuilder resolvedInstances = new StringBuilder();
+ StringBuilder resolvedURIPaths = new StringBuilder();
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
nominalInstanceCal = (Calendar) initInstance.clone();
@@ -255,13 +307,19 @@ public class CoordELFunctions {
pathWithDoneFlag += "/" + doneFlag;
}
if (isPathAvailable(pathWithDoneFlag, user, null, conf)) {
- XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag);
- if (available == n) {
- XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag);
+ LOG.debug("Found future(" + available + "): " + pathWithDoneFlag);
+ if (available == endOffset) {
+ LOG.debug("Matched future(" + available + "): " + pathWithDoneFlag);
resolved = true;
- retVal = DateUtils.formatDateOozieTZ(nominalInstanceCal);
- eval.setVariable("resolved_path", uriPath);
+ resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
+ resolvedURIPaths.append(uriPath);
+ retVal = resolvedInstances.toString();
+ eval.setVariable("resolved_path", resolvedURIPaths.toString());
break;
+ } else if (available >= startOffset) {
+ LOG.debug("Matched future(" + available + "): " + pathWithDoneFlag);
+ resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
+ resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
}
available++;
}
@@ -277,7 +335,12 @@ public class CoordELFunctions {
// return unchanged future function with variable 'is_resolved'
// to 'false'
eval.setVariable("is_resolved", Boolean.FALSE);
- retVal = "${coord:future(" + n + ", " + instance + ")}";
+ if (startOffset == endOffset) {
+ retVal = "${coord:future(" + startOffset + ", " + instance + ")}";
+ }
+ else {
+ retVal = "${coord:futureRange(" + startOffset + ", " + endOffset + ", " + instance + ")}";
+ }
}
else {
eval.setVariable("is_resolved", Boolean.TRUE);
@@ -538,6 +601,29 @@ public class CoordELFunctions {
}
/**
+ * Determine the date-time in Oozie processing timezone of latest available dataset instances
+ * from start to end offsets from the nominal time. <p/> It depends
+ * on: <p/> 1. Data set frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST
+ * <p/> 4. End Day/Month flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of
+ * dataset's directory
+ *
+ * @param start :start instance offset <p/> domain: start > 0, start is integer
+ * @param end :end instance offset <p/> domain: end > 0, end is integer
+ * @return date-time in Oozie processing timezone of the instances from start to end offsets
+ * delimited by comma. <p/> returns 'null' means start offset instance is
+ * earlier than Initial-Instance of DS
+ * @throws Exception
+ */
+ public static String ph3_coord_latestRange(int start, int end) throws Exception {
+ if (isSyncDataSet()) {// For Sync Dataset
+ return coord_latestRange_sync(start, end);
+ }
+ else {
+ throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
+ }
+ }
+
+ /**
* Configure an evaluator with data set and application specific information. <p/> Helper method of associating
* dataset and application object
*
@@ -618,11 +704,27 @@ public class CoordELFunctions {
return ph1_coord_future_echo(n, instance);
}
+ public static String ph1_coord_latestRange_echo(String start, String end) {
+ return echoUnResolved("latestRange", start + ", " + end);
+ }
+
+ public static String ph2_coord_latestRange_echo(String start, String end) {
+ return ph1_coord_latestRange_echo(start, end);
+ }
+
+ public static String ph1_coord_futureRange_echo(String start, String end, String instance) {
+ return echoUnResolved("futureRange", start + ", " + end + ", " + instance);
+ }
+
+ public static String ph2_coord_futureRange_echo(String start, String end, String instance) {
+ return ph1_coord_futureRange_echo(start, end, instance);
+ }
+
public static String ph1_coord_dataIn_echo(String n) {
ELEvaluator eval = ELEvaluator.getCurrent();
String val = (String) eval.getVariable("oozie.dataname." + n);
if (val == null || val.equals("data-in") == false) {
- XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid");
+ LOG.error("data_in_name " + n + " is not valid");
throw new RuntimeException("data_in_name " + n + " is not valid");
}
return echoUnResolved("dataIn", "'" + n + "'");
@@ -632,7 +734,7 @@ public class CoordELFunctions {
ELEvaluator eval = ELEvaluator.getCurrent();
String val = (String) eval.getVariable("oozie.dataname." + n);
if (val == null || val.equals("data-out") == false) {
- XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
+ LOG.error("data_out_name " + n + " is not valid");
throw new RuntimeException("data_out_name " + n + " is not valid");
}
return echoUnResolved("dataOut", "'" + n + "'");
@@ -739,8 +841,9 @@ public class CoordELFunctions {
int[] instCount = new int[1];// used as pass by ref
Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
if (nominalInstanceCal == null) {
- XLog.getLog(CoordELFunctions.class)
- .warn("If the initial instance of the dataset is later than the nominal time, an empty string is returned. This means that no data is available at the current-instance specified by the user and the user could try modifying his initial-instance to an earlier time.");
+ LOG.warn("If the initial instance of the dataset is later than the nominal time, an empty string is"
+ + " returned. This means that no data is available at the current-instance specified by the user"
+ + " and the user could try modifying his initial-instance to an earlier time.");
return "";
}
nominalInstanceCal = getInitialInstanceCal();
@@ -748,8 +851,10 @@ public class CoordELFunctions {
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount);
if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
- XLog.getLog(CoordELFunctions.class)
- .warn("If the initial instance of the dataset is later than the current-instance specified, such as coord:current({0}) in this case, an empty string is returned. This means that no data is available at the current-instance specified by the user and the user could try modifying his initial-instance to an earlier time.", n);
+ LOG.warn("If the initial instance of the dataset is later than the current-instance specified, such as"
+ + " coord:current({0}) in this case, an empty string is returned. This means that no data is"
+ + " available at the current-instance specified by the user and the user could try modifying his"
+ + " initial-instance to an earlier time.", n);
return "";
}
String str = DateUtils.formatDateOozieTZ(nominalInstanceCal);
@@ -804,7 +909,7 @@ public class CoordELFunctions {
Calendar nominalInstanceCal = getInitialInstanceCal();
nominalInstanceCal.add(freqUnit.getCalendarUnit(), freq * freqCount);
if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
- XLog.getLog(CoordELFunctions.class).warn("If the initial instance of the dataset is later than the offset instance"
+ LOG.warn("If the initial instance of the dataset is later than the offset instance"
+ " specified, such as coord:offset({0}, {1}) in this case, an empty string is returned. This means that no"
+ " data is available at the offset instance specified by the user and the user could try modifying his"
+ " initial-instance to an earlier time.", n, timeUnit);
@@ -824,9 +929,17 @@ public class CoordELFunctions {
* @throws Exception
*/
private static String coord_latest_sync(int offset) throws Exception {
- if (offset > 0) {
+ return coord_latestRange_sync(offset, offset);
+ }
+
+ private static String coord_latestRange_sync(int startOffset, int endOffset) throws Exception {
+ if (startOffset > 0) {
throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0"
- + offset);
+ + startOffset);
+ }
+ if (endOffset > 0) {
+ throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0"
+ + endOffset);
}
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
@@ -834,6 +947,8 @@ public class CoordELFunctions {
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
+ StringBuilder resolvedInstances = new StringBuilder();
+ StringBuilder resolvedURIPaths = new StringBuilder();
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
@@ -858,13 +973,19 @@ public class CoordELFunctions {
pathWithDoneFlag += "/" + doneFlag;
}
if (isPathAvailable(pathWithDoneFlag, user, null, conf)) {
- XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag);
- if (available == offset) {
- XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
+ LOG.debug("Found latest(" + available + "): " + pathWithDoneFlag);
+ if (available == endOffset) {
+ LOG.debug("Matched latest(" + available + "): " + pathWithDoneFlag);
resolved = true;
- retVal = DateUtils.formatDateOozieTZ(nominalInstanceCal);
- eval.setVariable("resolved_path", uriPath);
+ resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
+ resolvedURIPaths.append(uriPath);
+ retVal = resolvedInstances.toString();
+ eval.setVariable("resolved_path", resolvedURIPaths.toString());
break;
+ } else if (available <= startOffset) {
+ LOG.debug("Matched latest(" + available + "): " + pathWithDoneFlag);
+ resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
+ resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
}
available--;
@@ -880,7 +1001,12 @@ public class CoordELFunctions {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable("is_resolved", Boolean.FALSE);
- retVal = "${coord:latest(" + offset + ")}";
+ if (startOffset == endOffset) {
+ retVal = "${coord:latest(" + startOffset + ")}";
+ }
+ else {
+ retVal = "${coord:latestRange(" + startOffset + "," + endOffset + ")}";
+ }
}
else {
eval.setVariable("is_resolved", Boolean.TRUE);
@@ -1287,7 +1413,7 @@ public class CoordELFunctions {
cal = getCurrentInstance(getActionCreationtime(eval), null, eval);
}
if (cal == null) {
- XLog.getLog(CoordELFunctions.class).warn("If the initial instance of the dataset is later than the nominal time, an"
+ LOG.warn("If the initial instance of the dataset is later than the nominal time, an"
+ " empty string is returned. This means that no data is available at the offset instance specified by the user"
+ " and the user could try modifying his or her initial-instance to an earlier time.");
return null;
Modified: oozie/branches/branch-3.3/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/resources/oozie-default.xml?rev=1442249&r1=1442248&r2=1442249&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/resources/oozie-default.xml (original)
+++ oozie/branches/branch-3.3/core/src/main/resources/oozie-default.xml Mon Feb 4 18:02:40 2013
@@ -684,7 +684,9 @@
coord:current=org.apache.oozie.coord.CoordELFunctions#ph1_coord_current_echo,
coord:offset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_offset_echo,
coord:latest=org.apache.oozie.coord.CoordELFunctions#ph1_coord_latest_echo,
+ coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_latestRange_echo,
coord:future=org.apache.oozie.coord.CoordELFunctions#ph1_coord_future_echo,
+ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_futureRange_echo,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user
@@ -830,7 +832,9 @@
coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current,
coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset,
coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
+ coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo,
coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,
+ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo,
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,
@@ -883,7 +887,9 @@
coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current_echo,
coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset_echo,
coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
+ coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo,
coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,
+ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo,
coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user
@@ -975,7 +981,9 @@
coord:daysInMonth=org.apache.oozie.coord.CoordELFunctions#ph3_coord_daysInMonth,
coord:tzOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_tzOffset,
coord:latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest,
+ coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latestRange,
coord:future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future,
+ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph3_coord_futureRange,
coord:dataIn=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dataIn,
coord:dataOut=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dataOut,
coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime,
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java?rev=1442249&r1=1442248&r2=1442249&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java Mon Feb 4 18:02:40 2013
@@ -64,7 +64,7 @@ public class TestCoordActionInputCheckXC
services = new Services();
services.init();
cleanUpDBTables();
- TZ = (getProcessingTZ().equals(DateUtils.OOZIE_PROCESSING_TIMEZONE_DEFAULT))
+ TZ = (getProcessingTZ().equals(DateUtils.OOZIE_PROCESSING_TIMEZONE_DEFAULT))
? "Z" : getProcessingTZ().substring(3);
}
@@ -113,10 +113,10 @@ public class TestCoordActionInputCheckXC
CoordinatorActionBean action1 = addRecordToCoordActionTableForWaiting(job.getId(), 1,
CoordinatorAction.Status.WAITING, "coord-action-for-action-input-check.xml");
- createDir(getTestCaseDir() + "/2009/29/");
- createDir(getTestCaseDir() + "/2009/22/");
- createDir(getTestCaseDir() + "/2009/15/");
- createDir(getTestCaseDir() + "/2009/08/");
+ createDir(getTestCaseDir() + "/2009/01/29/");
+ createDir(getTestCaseDir() + "/2009/01/22/");
+ createDir(getTestCaseDir() + "/2009/01/15/");
+ createDir(getTestCaseDir() + "/2009/01/08/");
final MyCoordActionInputCheckXCommand callable1 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "1");
final MyCoordActionInputCheckXCommand callable2 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "2");
@@ -147,8 +147,8 @@ public class TestCoordActionInputCheckXC
Date endTime = DateUtils.parseDateOozieTZ("2009-02-02T23:59" + TZ);
CoordinatorJobBean job = addRecordToCoordJobTable(jobId, startTime, endTime);
new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
- createDir(getTestCaseDir() + "/2009/29/");
- createDir(getTestCaseDir() + "/2009/15/");
+ createDir(getTestCaseDir() + "/2009/01/29/");
+ createDir(getTestCaseDir() + "/2009/01/15/");
new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
checkCoordAction(job.getId() + "@1");
}
@@ -164,9 +164,10 @@ public class TestCoordActionInputCheckXC
CoordinatorJobBean job = addRecordToCoordJobTable(jobId, startTime, endTime);
new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
- // providing some of the dataset dirs required as per coordinator specification - /2009/12, /2009/05, /2009/29, /2009/22
- createDir(getTestCaseDir() + "/2009/12/");
- createDir(getTestCaseDir() + "/2009/29/");
+ // providing some of the dataset dirs required as per coordinator
+ // specification - /2009/02/12, /2009/02/05, /2009/01/29, /2009/01/22
+ createDir(getTestCaseDir() + "/2009/02/12/");
+ createDir(getTestCaseDir() + "/2009/01/29/");
new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
CoordinatorActionBean action = null;
@@ -180,23 +181,96 @@ public class TestCoordActionInputCheckXC
// Missing dependencies recorded by the coordinator action after input check
String missDepsOrder = action.getMissingDependencies();
- // Expected missing dependencies are /2009/05, /2009/29, and /2009/22.
+ // Expected missing dependencies are /2009/02/05, /2009/01/29, and /2009/01/22.
- int index = missDepsOrder.indexOf("/2009/12");
+ int index = missDepsOrder.indexOf("/2009/02/12");
if( index >= 0) {
fail("Dependency should be available! current list: " + missDepsOrder);
}
- // Case when /2009/29 exists but checking stops since dataset synchronously expected before i.e. /2009/05 is missing
- index = missDepsOrder.indexOf("/2009/29");
+ // Case when /2009/01/29 exists but checking stops since dataset synchronously expected before i.e. /2009/02/05 is missing
+ index = missDepsOrder.indexOf("/2009/01/29");
if( index < 0) {
fail("Data should have been in missing dependency list! current list: " + missDepsOrder);
}
- index = missDepsOrder.indexOf("/2009/05");
+ index = missDepsOrder.indexOf("/2009/02/05");
if( index < 0) {
fail("Data should have been in missing dependency list! current list: " + missDepsOrder);
}
}
+ public void testActionInputCheckLatest() throws Exception {
+ String jobId = "0000000-" + new Date().getTime() + "-TestCoordActionInputCheckXCommand-C";
+ Date startTime = DateUtils.parseDateOozieTZ("2009-02-15T23:59" + TZ);
+ Date endTime = DateUtils.parseDateOozieTZ("2009-02-16T23:59" + TZ);
+ CoordinatorJobBean job = addRecordToCoordJobTable(jobId, startTime, endTime, "latest");
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+
+ new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
+ CoordinatorActionBean action = null;
+ JPAService jpaService = Services.get().get(JPAService.class);
+ try {
+ action = jpaService.execute(new CoordActionGetJPAExecutor(job.getId() + "@1"));
+ }
+ catch (JPAExecutorException se) {
+ fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
+ }
+
+ assertEquals(";${coord:latestRange(-3,0)}", action.getMissingDependencies());
+
+ // providing some of the dataset dirs required as per coordinator specification with holes
+ createDir(getTestCaseDir() + "/2009/02/12/");
+ createDir(getTestCaseDir() + "/2009/02/05/");
+ createDir(getTestCaseDir() + "/2009/01/22/");
+ createDir(getTestCaseDir() + "/2009/01/08/");
+
+ new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
+ try {
+ action = jpaService.execute(new CoordActionGetJPAExecutor(job.getId() + "@1"));
+ }
+ catch (JPAExecutorException se) {
+ fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
+ }
+
+ assertEquals("", action.getMissingDependencies());
+ }
+
+ public void testActionInputCheckFuture() throws Exception {
+ String jobId = "0000000-" + new Date().getTime() + "-TestCoordActionInputCheckXCommand-C";
+ Date startTime = DateUtils.parseDateOozieTZ("2009-02-15T23:59" + TZ);
+ Date endTime = DateUtils.parseDateOozieTZ("2009-02-16T23:59" + TZ);
+ CoordinatorJobBean job = addRecordToCoordJobTable(jobId, startTime, endTime, "future");
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+
+ // providing some of the dataset dirs required as per coordinator specification with holes
+ createDir(getTestCaseDir() + "/2009/02/12/");
+ createDir(getTestCaseDir() + "/2009/02/26/");
+ createDir(getTestCaseDir() + "/2009/03/05/");
+ createDir(getTestCaseDir() + "/2009/03/26/"); //limit is 5. So this should be ignored
+
+ new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
+ CoordinatorActionBean action = null;
+ JPAService jpaService = Services.get().get(JPAService.class);
+ try {
+ action = jpaService.execute(new CoordActionGetJPAExecutor(job.getId() + "@1"));
+ }
+ catch (JPAExecutorException se) {
+ fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
+ }
+
+ assertEquals(";${coord:futureRange(0,3,'5')}", action.getMissingDependencies());
+
+ createDir(getTestCaseDir() + "/2009/03/12/");
+
+ new CoordActionInputCheckXCommand(job.getId() + "@1", job.getId()).call();
+ try {
+ action = jpaService.execute(new CoordActionGetJPAExecutor(job.getId() + "@1"));
+ }
+ catch (JPAExecutorException se) {
+ fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
+ }
+
+ assertEquals("", action.getMissingDependencies());
+ }
/**
* Testing a non existing namenode path
*
@@ -302,7 +376,8 @@ public class TestCoordActionInputCheckXC
CoordinatorAction.Status status, String resourceXmlName) throws Exception {
CoordinatorActionBean action = createCoordAction(jobId, actionNum, status, resourceXmlName, 0, TZ);
String testDir = getTestCaseDir();
- String missDeps = "file://#testDir/2009/29/_SUCCESS#file://#testDir/2009/22/_SUCCESS#file://#testDir/2009/15/_SUCCESS#file://#testDir/2009/08/_SUCCESS";
+ String missDeps = "file://#testDir/2009/01/29/_SUCCESS#file://#testDir/2009/01/22/_SUCCESS"
+ + "#file://#testDir/2009/01/15/_SUCCESS#file://#testDir/2009/01/08/_SUCCESS";
missDeps = missDeps.replaceAll("#testDir", testDir);
action.setMissingDependencies(missDeps);
@@ -321,6 +396,11 @@ public class TestCoordActionInputCheckXC
}
private CoordinatorJobBean addRecordToCoordJobTable(String jobId, Date start, Date end) throws CommandException {
+ return addRecordToCoordJobTable(jobId, start, end, "current");
+ }
+
+ private CoordinatorJobBean addRecordToCoordJobTable(String jobId, Date start, Date end,
+ String dataInType) throws CommandException {
CoordinatorJobBean coordJob = new CoordinatorJobBean();
coordJob.setId(jobId);
coordJob.setAppName("testApp");
@@ -359,16 +439,22 @@ public class TestCoordActionInputCheckXC
appXml += "<input-events>";
appXml += "<data-in name='A' dataset='a'>";
appXml += "<dataset name='a' frequency='7' initial-instance='2009-01-01T01:00" + TZ + "' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>";
- appXml += "<uri-template>file://" + testDir + "/${YEAR}/${DAY}</uri-template>";
+ appXml += "<uri-template>file://" + testDir + "/${YEAR}/${MONTH}/${DAY}</uri-template>";
appXml += "</dataset>";
- appXml += "<start-instance>${coord:current(-3)}</start-instance>";
- appXml += "<end-instance>${coord:current(0)}</end-instance>";
+ if (dataInType.equals("future")) {
+ appXml += "<start-instance>${coord:" + dataInType + "(0,5)}</start-instance>";
+ appXml += "<end-instance>${coord:" + dataInType + "(3,5)}</end-instance>";
+ }
+ else {
+ appXml += "<start-instance>${coord:" + dataInType + "(-3)}</start-instance>";
+ appXml += "<end-instance>${coord:" + dataInType + "(0)}</end-instance>";
+ }
appXml += "</data-in>";
appXml += "</input-events>";
appXml += "<output-events>";
appXml += "<data-out name='LOCAL_A' dataset='local_a'>";
appXml += "<dataset name='local_a' frequency='7' initial-instance='2009-01-01T01:00" + TZ + "' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>";
- appXml += "<uri-template>file://" + testDir + "/${YEAR}/${DAY}</uri-template>";
+ appXml += "<uri-template>file://" + testDir + "/${YEAR}/${MONTH}/${DAY}</uri-template>";
appXml += "</dataset>";
appXml += "<start-instance>${coord:current(-3)}</start-instance>";
appXml += "<instance>${coord:current(0)}</instance>";
@@ -416,10 +502,10 @@ public class TestCoordActionInputCheckXC
JPAService jpaService = Services.get().get(JPAService.class);
CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
System.out.println("missingDeps " + action.getMissingDependencies() + " Xml " + action.getActionXml());
- if (action.getMissingDependencies().indexOf("/2009/29/") >= 0) {
+ if (action.getMissingDependencies().indexOf("/2009/01/29/") >= 0) {
fail("directory should be resolved :" + action.getMissingDependencies());
}
- if (action.getMissingDependencies().indexOf("/2009/15/") < 0) {
+ if (action.getMissingDependencies().indexOf("/2009/01/15/") < 0) {
fail("directory should NOT be resolved :" + action.getMissingDependencies());
}
}
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442249&r1=1442248&r2=1442249&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 18:02:40 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-1073 Optimize latest and future EL resolution in case of start-instance and end-instance (rohini via virag)
OOZIE-816 Add Support for Hadoop 1.1.1 (zhujinwei and harsh via harsh)
OOZIE-1106 latest and future function do not work correctly when oozie processing timezone is non UTC (rohini via tucu)
OOZIE-1102 Update Oozie README.txt to have the TLP mailing list and links (jaoki via rkanter)