You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2018/03/27 08:18:01 UTC
oozie git commit: OOZIE-2726 Flaky test due to daylight saving
changes (sasishsaley, andras.piros via gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master 77cd1fe20 -> 91c3d0c02
OOZIE-2726 Flaky test due to daylight saving changes (sasishsaley, andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/91c3d0c0
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/91c3d0c0
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/91c3d0c0
Branch: refs/heads/master
Commit: 91c3d0c02bbd4ff7c8c115ecf676fe1559fa6d03
Parents: 77cd1fe
Author: Gezapeti Cseh <ge...@apache.org>
Authored: Tue Mar 27 10:17:54 2018 +0200
Committer: Gezapeti Cseh <ge...@apache.org>
Committed: Tue Mar 27 10:17:54 2018 +0200
----------------------------------------------------------------------
.../CoordMaterializeTransitionXCommand.java | 19 +-
.../command/coord/DaylightOffsetCalculator.java | 92 ++++++
.../TestCoordMaterializeTransitionXCommand.java | 291 +++++++++++--------
.../coord/TestDaylightOffsetCalculator.java | 126 ++++++++
release-log.txt | 1 +
5 files changed, 408 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
index 2b91253..b4e62e9 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
@@ -35,7 +35,6 @@ import org.apache.oozie.command.PreconditionException;
import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
import org.apache.oozie.coord.CoordUtils;
import org.apache.oozie.coord.TimeUnit;
-import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluatorUtil;
import org.apache.oozie.executor.jpa.BatchQueryExecutor;
import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
import org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor;
@@ -223,7 +222,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo
if (endMatdTime.compareTo(jobEndTime) > 0) {
endMatdTime = jobEndTime;
}
-
LOG.debug("Materializing coord job id=" + jobId + ", start=" + DateUtils.formatDateOozieTZ(startMatdTime) + ", end=" + DateUtils.formatDateOozieTZ(endMatdTime)
+ ", window=" + materializationWindow);
}
@@ -457,6 +455,9 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo
}
boolean firstMater = true;
+
+ end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end);
+
while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) {
if (pause != null && effStart.compareTo(pause) >= 0) {
break;
@@ -563,8 +564,12 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo
job.setStatus(Job.Status.RUNNING);
}
job.setPending();
+ Calendar end = Calendar.getInstance();
+ end.setTime(jobEndTime);
+
+ end = calculateEndTimeWithDSTOffset(end);
- if (jobEndTime.compareTo(endMatdTime) <= 0) {
+ if (end.getTime().compareTo(endMatdTime) <= 0) {
LOG.info("[" + job.getId() + "]: all actions have been materialized, set pending to true");
// set doneMaterialization to true when materialization is done
job.setDoneMaterialization();
@@ -574,6 +579,14 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo
job.setNextMaterializedTime(endMatdTime);
}
+ private Calendar calculateEndTimeWithDSTOffset(final Calendar endTime) {
+ final TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
+ final Calendar start = Calendar.getInstance(appTz);
+ start.setTime(startMatdTime);
+
+ return new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, endTime);
+ }
+
/* (non-Javadoc)
* @see org.apache.oozie.command.XCommand#getKey()
*/
http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java b/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java
new file mode 100644
index 0000000..79afe18
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.command.coord;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+
+/**
+ * Calculates daylight offset for a given {@code target} {@link Calendar}, given a {@link TimeZone}, and both
+ * {@link #startMatdTime} and {@link #endMatdTime} {@link Calendar} instances.
+ */
+class DaylightOffsetCalculator {
+ private final Date startMatdTime;
+ private final Date endMatdTime;
+
+ DaylightOffsetCalculator(final Date startMatdTime, final Date endMatdTime) {
+ this.startMatdTime = startMatdTime;
+ this.endMatdTime = endMatdTime;
+ }
+
+ /**
+ * It adjusts {@code target} according to daylight saving time difference between {@link #startMatdTime} and
+ * {@link #endMatdTime}.
+ * <ul>
+ * <li>if {@link #startMatdTime} is in DST (e.g. PDT) and {@link #endMatdTime} is in non-DST (e.g. PST), then we need to
+ * add {@link Calendar#DST_OFFSET} to {@code target}</li>
+ * <li>if {@link #startMatdTime} is in non-DST (e.g. PST) and {@link #endMatdTime} is in DST (e.g. PDT), then we need to
+ * subtract {@link Calendar#DST_OFFSET} to {@code target}
+ * <li>otherwise, we do not adjust {@code target} and simply return
+ * </ul>
+ *
+ * @param tz {@link TimeZone} for {@code target}
+ * @param target the {@link Calendar} to modify
+ * @return adjusted {@code target} according to DST offset between {@link #startMatdTime} and {@link #endMatdTime}
+ */
+ Calendar calculate(final TimeZone tz, final Calendar target) {
+ final Calendar targetWithDSTOffset = Calendar.getInstance();
+ targetWithDSTOffset.setTimeInMillis(target.getTime().getTime() + getDSTOffset(tz, startMatdTime, endMatdTime));
+
+ return targetWithDSTOffset;
+ }
+
+ /**
+ * Calculates daylight saving time difference between {@code beginDate} and {@code endDate}.
+ * <ul>
+ * <li>if {@code beginDate} is in DST (e.g. PDT) and {@code endDate} is in non-DST (e.g. PST), then we need to return
+ * {@link Calendar#DST_OFFSET}</li>
+ * <li>if {@code beginDate} is in non-DST (e.g. PST) and {@code endDate} is in DST (e.g. PDT), then we need to return
+ * {@code -1 *} {@link Calendar#DST_OFFSET}</li>
+ * <li>otherwise, return {@code 0}</li>
+ * </ul>
+ *
+ * @param tz {@link TimeZone} for {@code beginDate} and {@code endDate}
+ * @param beginDate the beginning {@link Date} of the reference interval
+ * @param endDate the ending {@link Date} of the reference interval
+ * @return DST offset between {@code beginDate} and {@code endDate}
+ */
+ static long getDSTOffset(final TimeZone tz, final Date beginDate, final Date endDate) {
+ if (tz.inDaylightTime(beginDate) && !tz.inDaylightTime(endDate)) {
+ final Calendar cal = Calendar.getInstance(tz);
+ cal.setTime(beginDate);
+
+ return cal.get(Calendar.DST_OFFSET);
+ }
+
+ if (!tz.inDaylightTime(beginDate) && tz.inDaylightTime(endDate)) {
+ final Calendar cal = Calendar.getInstance(tz);
+ cal.setTime(endDate);
+
+ return -cal.get(Calendar.DST_OFFSET);
+ }
+
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
index 61bbbfe..323d34a 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
@@ -20,6 +20,7 @@ package org.apache.oozie.command.coord;
import java.io.File;
import java.sql.Timestamp;
+import java.text.ParseException;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
@@ -75,10 +76,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.parseDateOozieTZ("2009-03-06T010:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordAction(job.getId() + "@1");
}
+ private int hoursToSeconds(final int hours) {
+ return new Long(java.util.concurrent.TimeUnit.HOURS.toSeconds(hours)).intValue();
+ }
+
public void testActionMaterForHcatalog() throws Exception {
Services.get().destroy();
Services services = super.setupServicesForHCatalog();
@@ -87,7 +92,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z");
CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-hcat.xml",
CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
CoordinatorActionBean actionBean = getCoordAction(job.getId() + "@1");
assertEquals("file://dummyhdfs/2009/05/_SUCCESS" + CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR
+ "${coord:latestRange(-1,0)}", actionBean.getMissingDependencies());
@@ -106,7 +111,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-neg-hcat.xml",
CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
try {
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
fail("Expected Command exception but didn't catch any");
}
catch (CommandException e) {
@@ -125,7 +130,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z");
CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-relative.xml",
CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
}
public void testActionMaterWithCronFrequency1() throws Exception {
@@ -133,16 +138,17 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"10,20 * * * *");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:20Z")};
- checkCoordActionsNominalTime(job.getId(), 2, nominalTimes);
+ final int expectedNominalTimeCount = 2;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 2);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:10Z"));
}
catch (JPAExecutorException se) {
@@ -156,7 +162,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"10-20 * * * *");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:11Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:12Z"),
@@ -168,13 +174,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
DateUtils.parseDateOozieTZ("2013-07-18T00:18Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:19Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"),};
- checkCoordActionsNominalTime(job.getId(), 11, nominalTimes);
+ final int expectedNominalTimeCount = 11;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 11);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:10Z"));
}
catch (JPAExecutorException se) {
@@ -188,14 +195,15 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"0/15 2 * 5-7 4,5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
- checkCoordActionsNominalTime(job.getId(), 0, new Date[]{});
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
+ final int expectedNominalTimeCount = 0;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, new Date[]{});
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 0);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T02:00Z"));
}
catch (JPAExecutorException se) {
@@ -209,19 +217,19 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"0/15 * * 5-7 4,5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:15Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:30Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:45Z"),};
- checkCoordActionsNominalTime(job.getId(), 4, nominalTimes);
-
+ final int expectedNominalTimeCount = 4;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 4);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"));
}
catch (JPAExecutorException se) {
@@ -235,17 +243,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"20/15 * * 5-7 4,5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:35Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:50Z"),};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 3);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:20Z"));
}
catch (JPAExecutorException se) {
@@ -259,17 +268,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"20");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:40Z"),};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 3);
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"));
}
catch (JPAExecutorException se) {
@@ -283,17 +293,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"20/15 * * 7,10 THU");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:35Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:50Z"),};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 3);
+ assertEquals(expectedNominalTimeCount, job.getLastActionNumber());
assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:20Z"));
}
catch (JPAExecutorException se) {
@@ -310,17 +321,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setMatThrottling(3);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"),
DateUtils.parseDateOozieTZ("2013-07-18T00:20Z")};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertFalse(job.isDoneMaterialization());
- assertEquals(3, job.getLastActionNumber());
+ assertEquals(expectedNominalTimeCount, job.getLastActionNumber());
assertEquals(DateUtils.parseDateOozieTZ("2013-07-18T00:30Z"), job.getNextMaterializedTime());
}
catch (JPAExecutorException se) {
@@ -342,19 +354,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setTimeUnit(Timeunit.CRON);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
final String startPlusOneHour = "2013-03-10T09:00Z";
final String startPlusTwoHours = "2013-03-10T10:00Z";
final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast),
DateUtils.parseDateOozieTZ(startPlusOneHour),
DateUtils.parseDateOozieTZ(startPlusTwoHours)};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
final JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertFalse("coordinator job shouldn't have yet been materialized", job.isDoneMaterialization());
- assertEquals("coordinator action count mismatch", 3, job.getLastActionNumber());
+ assertEquals("coordinator action count mismatch", expectedNominalTimeCount, job.getLastActionNumber());
final String startPlusThreeHours = "2013-03-10T11:00Z";
assertEquals("coordinator next materialization time mismatch",
DateUtils.parseDateOozieTZ(startPlusThreeHours), job.getNextMaterializedTime());
@@ -365,33 +378,41 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
}
}
- public void testCronFrequencyCatchupThrottleEqualsDuration() throws Exception {
+ public void testCronFrequencyCatchupThrottleEqualsDurationDSTChange() throws Exception {
final String startInThePast = "2013-03-10T08:00Z";
- final Date startTime = DateUtils.parseDateOozieTZ(startInThePast);
+ final Date startTimeBeforeDSTChange = DateUtils.parseDateOozieTZ(startInThePast);
final String startPlusTwoHoursAndSome = "2013-03-10T10:01Z";
- final Date endTime = DateUtils.parseDateOozieTZ(startPlusTwoHoursAndSome);
- CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- job.setNextMaterializedTime(startTime);
+ final Date endTimeAfterDSTChange = DateUtils.parseDateOozieTZ(startPlusTwoHoursAndSome);
+ CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP,
+ startTimeBeforeDSTChange,
+ endTimeAfterDSTChange,
+ false,
+ false,
+ 0);
+ job.setNextMaterializedTime(startTimeBeforeDSTChange);
job.setMatThrottling(3);
final String everyHour = "0 * * * *";
job.setFrequency(everyHour);
job.setTimeUnit(Timeunit.CRON);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
+
final String startPlusOneHour = "2013-03-10T09:00Z";
- final String startPlusTwoHours = "2013-03-10T10:00Z";
- final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast),
- DateUtils.parseDateOozieTZ(startPlusOneHour),
- DateUtils.parseDateOozieTZ(startPlusTwoHours)};
- checkCoordActionsNominalTime(job.getId(), 3, nominalTimes);
+ final Date[] nominalTimesWithDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast),
+ DateUtils.parseDateOozieTZ(startPlusOneHour)};
+ checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithDSTChange);
+
+ checkTwoActionsAfterCatchup(job);
+ }
+ private void checkTwoActionsAfterCatchup(CoordinatorJobBean job) throws ParseException {
try {
final JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue("coordinator job should have already been materialized", job.isDoneMaterialization());
- assertEquals("coordinator action count mismatch", 3, job.getLastActionNumber());
- final String startPlusThreeHours = "2013-03-10T11:00Z";
+ assertEquals("coordinator action count mismatch", 2, job.getLastActionNumber());
+ final String startPlusThreeHours = "2013-03-10T10:00Z";
assertEquals("coordinator next materialization time mismatch",
DateUtils.parseDateOozieTZ(startPlusThreeHours), job.getNextMaterializedTime());
}
@@ -401,7 +422,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
}
}
- public void testCronFrequencyCatchupThrottleMoreThanDuration() throws Exception {
+ public void testCronFrequencyCatchupThrottleMoreThanDurationNoDSTChange() throws Exception {
final String startInThePast = "2013-03-10T08:00Z";
final Date startTime = DateUtils.parseDateOozieTZ(startInThePast);
final String startPlusOneHourAndSome = "2013-03-10T09:01Z";
@@ -414,25 +435,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setTimeUnit(Timeunit.CRON);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
+
final String startPlusOneHour = "2013-03-10T09:00Z";
- final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast),
+ final Date[] nominalTimesWithoutDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast),
DateUtils.parseDateOozieTZ(startPlusOneHour)};
- checkCoordActionsNominalTime(job.getId(), 2, nominalTimes);
+ checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithoutDSTChange);
- try {
- final JPAService jpaService = Services.get().get(JPAService.class);
- job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
- assertTrue("coordinator job should have already been materialized", job.isDoneMaterialization());
- assertEquals("coordinator action count mismatch", 2, job.getLastActionNumber());
- final String startPlusTwoHours = "2013-03-10T10:00Z";
- assertEquals("coordinator next materialization time mismatch",
- DateUtils.parseDateOozieTZ(startPlusTwoHours), job.getNextMaterializedTime());
- }
- catch (final JPAExecutorException se) {
- se.printStackTrace();
- fail("Job ID " + job.getId() + " was not stored properly in db");
- }
+ checkTwoActionsAfterCatchup(job);
}
public void testActionMaterWithDST1() throws Exception {
@@ -440,19 +450,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2013-03-10T12:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"0 * * * *");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600*4).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(4)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-03-10T08:00Z"),
DateUtils.parseDateOozieTZ("2013-03-10T09:00Z"),
DateUtils.parseDateOozieTZ("2013-03-10T10:00Z"),
- DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"),};
- checkCoordActionsNominalTime(job.getId(), 4, nominalTimes);
+ };
+ final int expectedNominalTimeCount = 3;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 4);
- assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-03-10T12:00Z"));
+ assertEquals(expectedNominalTimeCount, job.getLastActionNumber());
+ assertEquals(DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"), job.getNextMaterializedTime());
}
catch (JPAExecutorException se) {
se.printStackTrace();
@@ -465,19 +476,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2012-11-04T11:00Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null,
"0 * * * *");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600*4).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(4)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2012-11-04T07:00Z"),
DateUtils.parseDateOozieTZ("2012-11-04T08:00Z"),
DateUtils.parseDateOozieTZ("2012-11-04T09:00Z"),
- DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"),};
- checkCoordActionsNominalTime(job.getId(), 4, nominalTimes);
+ DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z") };
+ final int expectedNominalTimeCount = 5;
+ checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes);
try {
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertTrue(job.isDoneMaterialization());
- assertEquals(job.getLastActionNumber(), 4);
- assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z"));
+ assertEquals(job.getLastActionNumber(), expectedNominalTimeCount);
+ assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T12:00Z"));
}
catch (JPAExecutorException se) {
se.printStackTrace();
@@ -489,7 +501,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z");
Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T10:04Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2009-03-06T10:00Z")};
checkCoordActionsNominalTime(job.getId(), 1, nominalTimes);
}
@@ -499,7 +511,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z");
Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T10:08Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2009-03-06T10:00Z"),
DateUtils.parseDateOozieTZ("2009-03-06T10:05Z")};
checkCoordActionsNominalTime(job.getId(), 2, nominalTimes);
@@ -515,10 +527,10 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z");
Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T09:58Z");
final CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
- waitFor(1000*60, new Predicate() {
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
+ waitFor(60_000, new Predicate() {
public boolean evaluate() throws Exception {
- return (getStatus(job.getId()) == CoordinatorJob.Status.PAUSED?true:false);
+ return (getStatus(job.getId()) == CoordinatorJob.Status.PAUSED);
}
});
checkCoordActions(job.getId(), 0, CoordinatorJob.Status.PAUSED);
@@ -531,7 +543,8 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("5");
job.setTimeUnit(Timeunit.MINUTE);
job.setMatThrottling(20);
- String dryRunOutput = new CoordMaterializeTransitionXCommand(job, 3600, startTime, endTime).materializeActions(true);
+ String dryRunOutput = new CoordMaterializeTransitionXCommand(job, hoursToSeconds(1), startTime, endTime)
+ .materializeActions(true);
String[] actions = dryRunOutput.split("action for new instance");
assertEquals(3, actions.length -1);
for(int i = 1; i < actions.length; i++) {
@@ -545,7 +558,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date pauseTime = null;
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime,
pauseTime, 300, "5");
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordActionsTimeout(job.getId() + "@1", 300);
}
@@ -553,7 +566,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2009-02-03T23:59Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING);
}
@@ -561,7 +574,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2009-02-03T23:59Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordWaiting(job.getId(), job.getMatThrottling());
}
@@ -574,7 +587,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.parseDateOozieTZ("2099-02-01T01:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.PREP);
}
@@ -587,7 +600,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.toDate(new Timestamp(System.currentTimeMillis() + 180 * 1000));
Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING);
}
@@ -610,7 +623,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
+ "<workflow>" + "<app-path>hdfs:///tmp/workflows/</app-path>" + "</workflow>" + "</action>"
+ "</coordinator-app>";
CoordinatorJobBean job = addRecordToCoordJobTable(coordXml);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
JPAService jpaService = Services.get().get(JPAService.class);
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertEquals(CoordinatorJob.Status.FAILED, job.getStatus());
@@ -629,7 +642,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
Date startTime = DateUtils.toDate(new Timestamp(System.currentTimeMillis() + 360 * 1000));
Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z");
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.PREP);
}
@@ -653,7 +666,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("1");
job.setTimeUnitStr("DAY");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertEquals(new Date(startTime.getTime() + TIME_IN_DAY * 3), job.getNextMaterializedTime());
@@ -666,7 +679,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("1");
job.setTimeUnitStr("HOUR");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
assertEquals(new Date(startTime.getTime() + TIME_IN_HOURS * 10), job.getNextMaterializedTime());
@@ -679,13 +692,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("1");
job.setTimeUnitStr("DAY");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
// If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for
// that because startTime and endTime assume GMT
Date next = new Date(startTime.getTime() + TIME_IN_DAY * 3);
TimeZone tz = TimeZone.getTimeZone(job.getTimeZone());
- next.setTime(next.getTime() - getDSTOffset(tz, startTime, next));
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
assertEquals(next, job.getNextMaterializedTime());
// test with hours, time should not pass the current time.
@@ -697,13 +710,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("1");
job.setTimeUnitStr("DAY");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
// If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for
// that because startTime and endTime assume GMT
next = new Date(startTime.getTime() + TIME_IN_DAY);
tz = TimeZone.getTimeZone(job.getTimeZone());
- next.setTime(next.getTime() - getDSTOffset(tz, startTime, next));
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
assertEquals(next, job.getNextMaterializedTime());
// for current job in min, should not exceed hour windows
@@ -714,13 +727,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("5");
job.setTimeUnitStr("MINUTE");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
// If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for
// that because startTime and endTime assume GMT
next = new Date(startTime.getTime() + TIME_IN_HOURS);
tz = TimeZone.getTimeZone(job.getTimeZone());
- next.setTime(next.getTime() - getDSTOffset(tz, startTime, next));
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
assertEquals(next, job.getNextMaterializedTime());
// for current job in hour, should not exceed hour windows
@@ -731,27 +744,83 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setFrequency("1");
job.setTimeUnitStr("DAY");
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
// If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for
// that because startTime and endTime assume GMT
next = new Date(startTime.getTime() + TIME_IN_DAY);
tz = TimeZone.getTimeZone(job.getTimeZone());
- next.setTime(next.getTime() - getDSTOffset(tz, startTime, next));
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
assertEquals(next, job.getNextMaterializedTime());
+ // Case: job started in Daylight time, and materialization is in
+ // Standard time
+ startTime = getDaylightCalendar().getTime();
+ endTime = new Date(startTime.getTime() + TIME_IN_DAY * 3);
+ job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
+ job.setNextMaterializedTime(startTime);
+ job.setMatThrottling(10);
+ job.setFrequency("1");
+ job.setTimeUnitStr("DAY");
+ CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
+ job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
+ // If the startTime and endTime straddle a DST shift (the Coord is in
+ // "America/Los_Angeles"), then we need to adjust for
+ // that because startTime and endTime assume GMT
+ next = new Date(startTime.getTime() + TIME_IN_DAY * 3);
+ tz = TimeZone.getTimeZone(job.getTimeZone());
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
+
+ assertEquals(next, job.getNextMaterializedTime());
+
+ // Case: job started in Standard time, and materialization is in
+ // Daylight time
+ Calendar c = getStandardCalendar();
+ startTime = c.getTime();
+ endTime = new Date(startTime.getTime() + TIME_IN_DAY * 3);
+ job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0);
+ job.setNextMaterializedTime(startTime);
+ job.setMatThrottling(10);
+ job.setFrequency("1");
+ job.setTimeUnitStr("DAY");
+ CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
+ new CoordMaterializeTransitionXCommand(job, hoursToSeconds(1), startTime, endTime).call();
+ job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
+ // If the startTime and endTime straddle a DST shift (the Coord is in
+ // "America/Los_Angeles"), then we need to adjust for
+ // that because startTime and endTime assume GMT
+ next = new Date(startTime.getTime() + TIME_IN_DAY * 3);
+ tz = TimeZone.getTimeZone(job.getTimeZone());
+ next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next));
+ assertEquals(next, job.getNextMaterializedTime());
+
+ }
+
+ Calendar getDaylightCalendar() {
+ final Calendar daylight = Calendar.getInstance();
+ daylight.set(2012, 10, 2, 15, 28, 00);
+
+ return daylight;
+ }
+
+ Calendar getStandardCalendar() {
+ final Calendar standard = Calendar.getInstance();
+ standard.set(2013, 2, 9, 15, 28, 00);
+
+ return standard;
}
public void testLastOnlyMaterialization() throws Exception {
long now = System.currentTimeMillis();
- Date startTime = DateUtils.toDate(new Timestamp(now - 180 * 60 * 1000)); // 3 hours ago
- Date endTime = DateUtils.toDate(new Timestamp(now + 180 * 60 * 1000)); // 3 hours from now
+ Date startTime = DateUtils.toDate(new Timestamp(now - 180 * 60 * 1000)); // 3 secondsFromHours ago
+ Date endTime = DateUtils.toDate(new Timestamp(now + 180 * 60 * 1000)); // 3 secondsFromHours from now
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, -1, "10",
CoordinatorJob.Execution.LAST_ONLY);
// This would normally materialize the throttle amount and within a 1 hour window; however, with LAST_ONLY this should
// ignore those parameters and materialize everything in the past
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING);
CoordinatorActionBean.Status[] expectedStatuses = new CoordinatorActionBean.Status[19];
Arrays.fill(expectedStatuses, CoordinatorActionBean.Status.WAITING);
@@ -761,7 +830,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, -1, "10",
CoordinatorJob.Execution.LAST_ONLY);
// We're starting from "now" this time (i.e. present/future), so it should materialize things normally
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING);
expectedStatuses = new CoordinatorActionBean.Status[6];
Arrays.fill(expectedStatuses, CoordinatorActionBean.Status.WAITING);
@@ -771,15 +840,15 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
public void testCurrentTimeCheck() throws Exception {
long now = System.currentTimeMillis();
Date startTime = DateUtils.toDate(new Timestamp(now)); // now
- Date endTime = DateUtils.toDate(new Timestamp(now + 3 * 60 * 60 * 1000)); // 3 hours from now
+ Date endTime = DateUtils.toDate(new Timestamp(now + 3 * 60 * 60 * 1000)); // 3 secondsFromHours from now
CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "5",
20);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING);
job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
assertEquals(job.getLastActionNumber(), 12);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
// unfortunatily XCommand doesn't throw exception on precondition
// assertEquals(e.getErrorCode(), ErrorCode.E1100);
// assertTrue(e.getMessage().contains("Request is for future time. Lookup time is"));
@@ -833,7 +902,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setEndTime(endTime);
job.setMatThrottling(10);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
assertEquals(job.getLastActionNumber(), 3);
@@ -880,7 +949,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
job.setMatThrottling(10);
CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
- new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
assertEquals(4, job.getLastActionNumber());
@@ -1083,20 +1152,6 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
}
}
- private long getDSTOffset(TimeZone tz, Date d1, Date d2) {
- if (tz.inDaylightTime(d1) && !tz.inDaylightTime(d2)) {
- Calendar cal = Calendar.getInstance(tz);
- cal.setTime(d1);
- return cal.get(Calendar.DST_OFFSET);
- }
- if (!tz.inDaylightTime(d1) && tz.inDaylightTime(d2)) {
- Calendar cal = Calendar.getInstance(tz);
- cal.setTime(d2);
- return cal.get(Calendar.DST_OFFSET);
- }
- return 0;
- }
-
/**
* Test a coordinator SLA define EL functions as variable
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java b/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java
new file mode 100644
index 0000000..1427e08
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.command.coord;
+
+import org.apache.oozie.util.DateUtils;
+import org.junit.Test;
+
+import java.text.ParseException;
+import java.util.Calendar;
+import java.util.TimeZone;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDaylightOffsetCalculator {
+ private final static TimeZone TZ_DST = TimeZone.getTimeZone("America/Los_Angeles");
+ private final static TimeZone TZ_STANDARD = TimeZone.getTimeZone("Asia/Kolkata");
+
+ @Test
+ public void testCalculateBetweenNonDSTStartAndEndGivesZeroOffset() throws ParseException {
+ final DaylightOffsetCalculator nonDSTStartAndEndCalculator = new DaylightOffsetCalculator(
+ DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2014-01-18T00:00Z"));
+
+ final Calendar target = Calendar.getInstance();
+ target.setTime(DateUtils.parseDateOozieTZ("2014-01-18T00:00Z"));
+
+ assertEquals(nonDSTStartAndEndCalculator.calculate(TZ_DST, target), target);
+ }
+
+ @Test
+ public void testCalculateBetweenDSTStartAndStandardEndGivesPositiveOffset() throws ParseException {
+ final DaylightOffsetCalculator dstStartAndStandardEndCalculator = new DaylightOffsetCalculator(
+ DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2014-01-18T00:00Z"));
+
+ final Calendar standardEnd = Calendar.getInstance();
+ standardEnd.setTime(DateUtils.parseDateOozieTZ("2014-01-18T00:00Z"));
+
+ final Calendar standardEndPlusOneHour = (Calendar) standardEnd.clone();
+ standardEndPlusOneHour.add(Calendar.HOUR, 1);
+
+ assertEquals(dstStartAndStandardEndCalculator.calculate(TZ_DST, standardEnd), standardEndPlusOneHour);
+ }
+
+ @Test
+ public void testCalculateBetweenStandardStartAndDSTEndGivesNegativeOffset() throws ParseException {
+ final DaylightOffsetCalculator standardStartAndDSTEndCalculator = new DaylightOffsetCalculator(
+ DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"));
+
+ final Calendar standardEnd = Calendar.getInstance();
+ standardEnd.setTime(DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"));
+
+ final Calendar standardEndMinusOneHour = (Calendar) standardEnd.clone();
+ standardEndMinusOneHour.add(Calendar.HOUR, -1);
+
+ assertEquals(standardStartAndDSTEndCalculator.calculate(TZ_DST, standardEnd), standardEndMinusOneHour);
+ }
+
+ @Test
+ public void testCalculateGivenStandardTZGivesZeroOffset() throws ParseException {
+ final DaylightOffsetCalculator standardStartAndDSTEndCalculator = new DaylightOffsetCalculator(
+ DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"));
+
+ final Calendar dstStart = Calendar.getInstance();
+ dstStart.setTime(DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"));
+
+ final Calendar standardEnd = Calendar.getInstance();
+ standardEnd.setTime(DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"));
+
+ assertEquals(standardStartAndDSTEndCalculator.calculate(TZ_STANDARD, standardEnd), standardEnd);
+ }
+
+ @Test
+ public void testGetDSTOffsetOfNonDaylightChangeGivesZeroOffset() throws ParseException {
+ final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST,
+ DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2013-07-19T00:00Z"));
+
+ assertEquals("non-daylight change should give zero DST offset", 0L, dstOffset);
+ }
+
+ @Test
+ public void testGetDSTOffsetOfDaylightChangeFromDSTToStandardGivesPositiveOffset() throws ParseException {
+ final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST,
+ DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2014-01-19T00:00Z"));
+
+ assertEquals("daylight change from DST to standard should give positive DST offset", 3_600_000L, dstOffset);
+ }
+
+ @Test
+ public void testGetDSTOffsetOfDaylightChangeFromStandardToDSTGivesNegativeOffset() throws ParseException {
+ final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST,
+ DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2013-07-19T00:00Z"));
+
+ assertEquals("daylight change from standard to DST should give negative DST offset", -3_600_000L, dstOffset);
+ }
+
+ @Test
+ public void testGetDSTOffsetGivenStandardTZGivesZeroOffset() throws ParseException {
+ final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_STANDARD,
+ DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"),
+ DateUtils.parseDateOozieTZ("2013-07-19T00:00Z"));
+
+ assertEquals("daylight change from standard to DST should give negative DST offset", 0L, dstOffset);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d7e3057..84880ca 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-2726 Flaky test due to daylight saving changes (sasishsaley, andras.piros via gezapeti)
OOZIE-2645 Deprecate Instrumentation in favor of Metrics (andras.piros via gezapeti)
OOZIE-3150 Remove references to not present dependencies within NOTICE.txt (gezapeti via andras.piros)
OOZIE-3201 Typo in TestCoordActionInputCheckXCommand (gongchuanjie via andras.piros)