You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/08/17 04:00:20 UTC
falcon git commit: FALCON-2116 Queue defined in is
not getting used in actual retention job
Repository: falcon
Updated Branches:
refs/heads/master 7354f870b -> 3a7c993b5
FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention job
Author: sandeep <sa...@gmail.com>
Reviewers: @pallavi-rao
Closes #266 from sandeepSamudrala/FALCON-2116 and squashes the following commits:
a23fe90 [sandeep] FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention job
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3a7c993b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3a7c993b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3a7c993b
Branch: refs/heads/master
Commit: 3a7c993b55564d11ae79d25d32fd5fbd63ce4fe9
Parents: 7354f87
Author: sandeep <sa...@gmail.com>
Authored: Wed Aug 17 09:30:13 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Wed Aug 17 09:30:13 2016 +0530
----------------------------------------------------------------------
.../engine/oozie/retention/AgeBasedCoordinatorBuilder.java | 8 ++++++++
.../falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java | 3 ++-
.../src/test/resources/feed/fs-retention-lifecycle-feed.xml | 3 ++-
3 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
index 7a25d86..f7b31e0 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
@@ -41,6 +41,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
+import java.util.List;
import java.util.Properties;
/**
@@ -100,6 +101,13 @@ public final class AgeBasedCoordinatorBuilder {
Path buildPath = OozieBuilderUtils.getBuildPath(basePath, LifeCycle.EVICTION.getTag());
Properties props = OozieBuilderUtils.createCoordDefaultConfiguration(coordName, feed);
props.putAll(OozieBuilderUtils.getProperties(buildPath, coordName));
+ props.putAll(EntityUtil.getEntityProperties(feed));
+ props.put("queueName", FeedHelper.getLifecycleRetentionQueue(feed, cluster.getName()));
+ List<org.apache.falcon.entity.v0.feed.Property> retentionProperties =
+ FeedHelper.getLifecycle(feed, cluster.getName()).getRetentionStage().getProperties().getProperties();
+ for (org.apache.falcon.entity.v0.feed.Property retentionProperty : retentionProperties) {
+ props.put(retentionProperty.getName(), retentionProperty.getValue());
+ }
WORKFLOW workflow = new WORKFLOW();
String entityPath = wfProp.getProperty(OozieBuilderUtils.ENTITY_PATH);
http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index fde5532..08d3339 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -189,6 +189,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
assertLibExtensions(coord, "retention");
HashMap<String, String> props = getCoordProperties(coord);
Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION");
+ Assert.assertEquals(props.get("queueName"), "ageBasedDeleteQueue");
Assert.assertEquals(coord.getFrequency(), "${coord:hours(17)}");
Assert.assertEquals(coord.getEnd(), endTime);
Assert.assertEquals(coord.getTimezone(), "UTC");
@@ -197,7 +198,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
Assert.assertEquals(wfProps.get("feedNames"), lifecycleRetentionFeed.getName());
Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name()));
Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon");
- Assert.assertEquals(wfProps.get("queueName"), "retention");
+ Assert.assertEquals(wfProps.get("queueName"), "ageBasedDeleteQueue");
Assert.assertEquals(wfProps.get("limit"), "hours(2)");
Assert.assertEquals(wfProps.get("jobPriority"), "LOW");
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
index 2cadfe0..a230f15 100644
--- a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
+++ b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
@@ -45,12 +45,13 @@
<properties>
<property name="maxMaps" value="33" />
<property name="mapBandwidth" value="2" />
+ <property name="queueName" value="retentionQueue" />
</properties>
<lifecycle>
<retention-stage>
<frequency>hours(17)</frequency>
- <queue>retention</queue>
+ <queue>ageBasedDeleteQueue</queue>
<priority>LOW</priority>
<properties>
<property name="retention.policy.agebaseddelete.limit" value="hours(2)"></property>