You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by so...@apache.org on 2015/11/09 22:43:51 UTC
falcon git commit: FALCON-1569 Bug in setting the frequency of Feed
retention coordinator. Contributed by Sowmya Ramesh
Repository: falcon
Updated Branches:
refs/heads/master aa18bfdae -> d97052f34
FALCON-1569 Bug in setting the frequency of Feed retention coordinator. Contributed by Sowmya Ramesh
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/d97052f3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/d97052f3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/d97052f3
Branch: refs/heads/master
Commit: d97052f3423fae8213e4e3bf677ee21b632b82b0
Parents: aa18bfd
Author: Sowmya Ramesh <sr...@hortonworks.com>
Authored: Mon Nov 9 13:43:44 2015 -0800
Committer: Sowmya Ramesh <sr...@hortonworks.com>
Committed: Mon Nov 9 13:43:44 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../feed/FeedRetentionCoordinatorBuilder.java | 8 ++--
.../feed/OozieFeedWorkflowBuilderTest.java | 39 ++++++++++++++++++++
.../apache/falcon/unit/FalconUnitTestBase.java | 6 ++-
4 files changed, 50 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 586847e..5a964da 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -27,6 +27,8 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-1569 Bug in setting the frequency of Feed retention coordinator(Sowmya Ramesh)
+
FALCON-1578 post-processing action fails with javax.servlet.jsp.el.ELException(Venkat Ramachandran via Ajay Yadava)
FALCON-1579 post-processing action fails with javax.servlet.jsp.el.ELException (Sowmya Ramesh via Pallavi Rao)
http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
index 0809bb6..69ca2c3 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
@@ -24,7 +24,7 @@ import org.apache.falcon.LifeCycle;
import org.apache.falcon.Tag;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.FeedHelper;
-import org.apache.falcon.entity.v0.Frequency.TimeUnit;
+import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
@@ -34,6 +34,7 @@ import org.apache.falcon.oozie.OozieOrchestrationWorkflowBuilder;
import org.apache.falcon.oozie.coordinator.ACTION;
import org.apache.falcon.oozie.coordinator.COORDINATORAPP;
import org.apache.falcon.oozie.coordinator.WORKFLOW;
+import org.apache.falcon.util.DateUtil;
import org.apache.hadoop.fs.Path;
import java.util.Arrays;
@@ -67,8 +68,9 @@ public class FeedRetentionCoordinatorBuilder extends OozieCoordinatorBuilder<Fee
coord.setStart(SchemaHelper.formatDateUTC(new Date()));
}
coord.setTimezone(entity.getTimezone().getID());
- TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
- if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
+ Frequency entityFrequency = entity.getFrequency();
+ Frequency defaultFrequency = new Frequency("hours(24)");
+ if (DateUtil.getFrequencyInMillis(entityFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) {
coord.setFrequency("${coord:hours(6)}");
} else {
coord.setFrequency("${coord:days(1)}");
http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 5e93027..176a15e 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -29,6 +29,7 @@ import org.apache.falcon.entity.Storage;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.ClusterLocationType;
@@ -187,6 +188,44 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
}
@Test
+ public void testRetentionFrequency() throws Exception {
+ feed.setFrequency(new Frequency("minutes(36000)"));
+ buildCoordAndValidateFrequency("${coord:days(1)}");
+
+ feed.setFrequency(new Frequency("hours(2)"));
+ buildCoordAndValidateFrequency("${coord:hours(6)}");
+
+ feed.setFrequency(new Frequency("minutes(50)"));
+ buildCoordAndValidateFrequency("${coord:hours(6)}");
+
+ feed.setFrequency(new Frequency("days(1)"));
+ buildCoordAndValidateFrequency("${coord:days(1)}");
+ }
+
+ private void buildCoordAndValidateFrequency(final String expectedFrequency) throws Exception {
+ // retention on src cluster
+ OozieCoordinatorBuilder builder = OozieCoordinatorBuilder.get(feed, Tag.RETENTION);
+ List<Properties> srcCoords = builder.buildCoords(
+ srcCluster, new Path("/projects/falcon/"));
+ COORDINATORAPP srcCoord = getCoordinator(srcMiniDFS, srcCoords.get(0).getProperty(OozieEntityBuilder
+ .ENTITY_PATH));
+
+ // Assert src coord frequency
+ Assert.assertEquals(srcCoord.getFrequency(), expectedFrequency);
+
+ // retention on target cluster
+ OozieEntityBuilder entityBuilder = OozieEntityBuilder.get(feed);
+ Path bundlePath = new Path("/projects/falcon/");
+ entityBuilder.build(trgCluster, bundlePath);
+ BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath);
+ List<COORDINATOR> coords = bundle.getCoordinator();
+
+ COORDINATORAPP tgtCoord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath());
+ // Assert target coord frequency
+ Assert.assertEquals(tgtCoord.getFrequency(), expectedFrequency);
+ }
+
+ @Test
public void testReplicationCoordsForFSStorage() throws Exception {
OozieEntityBuilder builder = OozieEntityBuilder.get(feed);
Path bundlePath = new Path("/projects/falcon/");
http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
----------------------------------------------------------------------
diff --git a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
index bd81798..af5a3c5 100644
--- a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
+++ b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
@@ -350,11 +350,13 @@ public class FalconUnitTestBase {
public InstancesResult.WorkflowStatus getRetentionStatus(String feedName, String cluster) throws FalconException,
FalconCLIException {
Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName);
- Frequency.TimeUnit timeUnit = feedEntity.getFrequency().getTimeUnit();
+
+ Frequency feedFrequency = feedEntity.getFrequency();
+ Frequency defaultFrequency = new Frequency("hours(24)");
long endTimeInMillis = System.currentTimeMillis() + 30000;
String endTime = DateUtil.getDateFormatFromTime(endTimeInMillis);
long startTimeInMillis;
- if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes) {
+ if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) {
startTimeInMillis = endTimeInMillis - (6 * DateUtil.HOUR_IN_MILLIS);
} else {
startTimeInMillis = endTimeInMillis - (24 * DateUtil.HOUR_IN_MILLIS);