You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/01/14 11:53:00 UTC
[1/2] falcon git commit: FALCON-1677 Support re-tries for timed-out
instances (by Narayan Periwal)
Repository: falcon
Updated Branches:
refs/heads/master 0657257d8 -> 583651c00
FALCON-1677 Support re-tries for timed-out instances (by Narayan Periwal)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8dfc2c93
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8dfc2c93
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8dfc2c93
Branch: refs/heads/master
Commit: 8dfc2c9368c2be34a788b58cbe034b57090e0792
Parents: 0657257
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Thu Jan 14 15:26:38 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Jan 14 15:26:38 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../src/main/resources/hdfs-replication.properties | 1 +
.../resources/hive-disaster-recovery-secure.properties | 1 +
.../main/resources/hive-disaster-recovery.properties | 1 +
.../org/apache/falcon/recipe/RecipeToolOptions.java | 1 +
.../falcon/recipe/util/RecipeProcessBuilderUtils.java | 5 +++++
client/src/main/resources/process-0.1.xsd | 1 +
.../main/java/org/apache/falcon/entity/EntityUtil.java | 1 +
.../falcon/entity/parser/ProcessEntityParserTest.java | 9 +++++++++
docs/src/site/twiki/EntitySpecification.twiki | 13 ++++++++++---
.../HdfsRecipe/hive-disaster-recovery.properties | 1 +
.../HiveDrRecipe/hive-disaster-recovery.properties | 1 +
.../hive-disaster-recovery-secure.properties | 1 +
.../org/apache/falcon/rerun/handler/RetryHandler.java | 10 ++++++++--
src/conf/runtime.properties | 6 ++++++
webapp/src/test/resources/hdfs-replication.properties | 1 +
16 files changed, 50 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f9316a4..cf97ca2 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,8 @@ Proposed Release Version: 0.9
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-1677 Support re-tries for timed-out instances (Narayan Periwal via Pallavi Rao)
+
FALCON-1643 Add CLI option to display captured replication metrics(Peeyush Bishnoi via Ajay Yadava)
FALCON-1679 API to get type of scheduler(native/oozie) (Pallavi Rao)
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
index 7ef4095..4642835 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
@@ -52,6 +52,7 @@ falcon.recipe.process.frequency=minutes(5)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### ACL properties - Uncomment and change ACL if authorization is enabled
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
index df4a8fd..8d00bb5 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
@@ -50,6 +50,7 @@ falcon.recipe.process.frequency=minutes(60)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
##### Uncomment to add tags
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
index c2915fb..b14ec7c 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
@@ -48,6 +48,7 @@ falcon.recipe.process.frequency=minutes(60)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
##### Uncomment to add tags
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
index 0ece90b..cebf968 100644
--- a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
+++ b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
@@ -38,6 +38,7 @@ public enum RecipeToolOptions {
RETRY_POLICY("falcon.recipe.retry.policy", "Retry policy", false),
RETRY_DELAY("falcon.recipe.retry.delay", "Retry delay", false),
RETRY_ATTEMPTS("falcon.recipe.retry.attempts", "Retry attempts", false),
+ RETRY_ON_TIMEOUT("falcon.recipe.retry.onTimeout", "Retry onTimeout", false),
RECIPE_TAGS("falcon.recipe.tags", "Recipe tags", false),
RECIPE_ACL_OWNER("falcon.recipe.acl.owner", "Recipe acl owner", false),
RECIPE_ACL_GROUP("falcon.recipe.acl.group", "Recipe acl group", false),
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
index 9477bb4..7caacb5 100644
--- a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
+++ b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
@@ -175,6 +175,11 @@ public final class RecipeProcessBuilderUtils {
if (StringUtils.isNotEmpty(retryDelay)) {
processRetry.setDelay(Frequency.fromString(retryDelay));
}
+
+ String retryOnTimeout = recipeProperties.getProperty(RecipeToolOptions.RETRY_ON_TIMEOUT.getName());
+ if (StringUtils.isNotEmpty(retryOnTimeout)) {
+ processRetry.setOnTimeout(Boolean.valueOf(retryOnTimeout));
+ }
}
private static void bindNotificationProperties(final Notification processNotification,
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/client/src/main/resources/process-0.1.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/process-0.1.xsd b/client/src/main/resources/process-0.1.xsd
index 256a29f..9d7898f 100644
--- a/client/src/main/resources/process-0.1.xsd
+++ b/client/src/main/resources/process-0.1.xsd
@@ -344,6 +344,7 @@
</xs:restriction>
</xs:simpleType>
</xs:attribute>
+ <xs:attribute type="xs:boolean" name="onTimeout" use="optional" default="false"/>
</xs:complexType>
<xs:simpleType name="policy-type">
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
index f448d70..24dbf3d 100644
--- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
+++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
@@ -670,6 +670,7 @@ public final class EntityUtil {
"feed.retry.frequency", "minutes(5)")));
retry.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
.getProperty("feed.retry.policy", "exp-backoff")));
+ retry.setOnTimeout(Boolean.valueOf(RuntimeProperties.get().getProperty("feed.retry.onTimeout", "false")));
return retry;
case PROCESS:
Process process = (Process) entity;
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
index da22d56..7159966 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
@@ -271,6 +271,15 @@ public class ProcessEntityParserTest extends AbstractTestBase {
parser.parseAndValidate(process.toString());
}
+ @Test()
+ public void testRetryTimeout() throws FalconException {
+ Process process = parser
+ .parseAndValidate(ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML));
+ process.getRetry().setOnTimeout(new Boolean("true"));
+ parser.parseAndValidate(process.toString());
+ }
+
@Test(expectedExceptions = ValidationException.class)
public void testInvalidLateInputs() throws Exception {
Process process = parser
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/docs/src/site/twiki/EntitySpecification.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki
index c4f5f76..6f24d8f 100644
--- a/docs/src/site/twiki/EntitySpecification.twiki
+++ b/docs/src/site/twiki/EntitySpecification.twiki
@@ -897,12 +897,12 @@ Feeds with Hive table storage will send one more parameter apart from the genera
<verbatim>$input_filter</verbatim>
---+++ Retry
-Retry policy defines how the workflow failures should be handled. Three retry policies are defined: periodic, exp-backoff(exponential backoff) and final. Depending on the delay and number of attempts, the workflow is re-tried after specific intervals.
+Retry policy defines how the workflow failures should be handled. Three retry policies are defined: periodic, exp-backoff(exponential backoff) and final. Depending on the delay and number of attempts, the workflow is re-tried after specific intervals. If user sets the onTimeout attribute to "true", retries will happen for TIMED_OUT instances.
Syntax:
<verbatim>
<process name="[process name]">
...
- <retry policy=[retry policy] delay=[retry delay] attempts=[retry attempts]/>
+ <retry policy=[retry policy] delay=[retry delay] attempts=[retry attempts] onTimeout=[retry onTimeout]/>
...
</process>
</verbatim>
@@ -911,12 +911,19 @@ Examples:
<verbatim>
<process name="sample-process">
...
- <retry policy="periodic" delay="minutes(10)" attempts="3"/>
+ <retry policy="periodic" delay="minutes(10)" attempts="3" onTimeout="true"/>
...
</process>
</verbatim>
The workflow is re-tried after 10 mins, 20 mins and 30 mins. With exponential backoff, the workflow will be re-tried after 10 mins, 20 mins and 40 mins.
+To enable retries for instances for feeds, user will have to set the following properties in runtime.properties
+<verbatim>
+falcon.recipe.retry.policy=periodic
+falcon.recipe.retry.delay=minutes(30)
+falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
+<verbatim>
---+++ Late data
Late data handling defines how the late data should be handled. Each feed is defined with a late cut-off value which specifies the time till which late data is valid. For example, late cut-off of hours(6) means that data for nth hour can get delayed by upto 6 hours. Late data specification in process defines how this late data is handled.
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
index fb2a4fc..7c95db7 100644
--- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
+++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hive-disaster-recovery.properties
@@ -52,6 +52,7 @@ falcon.recipe.process.frequency=minutes(5)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### ACL properties - Uncomment and change ACL if authorization is enabled
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery.properties b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery.properties
index dd781a5..05e11f7 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery.properties
+++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery.properties
@@ -48,6 +48,7 @@ falcon.recipe.process.frequency=minutes(60)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
##### Uncomment to add tags
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure.properties
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure.properties b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure.properties
index da0bcd1..62d4827 100644
--- a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure.properties
+++ b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure.properties
@@ -50,6 +50,7 @@ falcon.recipe.process.frequency=minutes(60)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
##### Tag properties - An optional list of comma separated tags, Key Value Pairs, separated by comma
##### Uncomment to add tags
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryHandler.java
----------------------------------------------------------------------
diff --git a/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryHandler.java b/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryHandler.java
index 84cd93f..fe2ceda 100644
--- a/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryHandler.java
+++ b/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryHandler.java
@@ -106,9 +106,15 @@ public class RetryHandler<M extends DelayedQueue<RetryEvent>> extends
@Override
public void onFailure(WorkflowExecutionContext context) throws FalconException {
- // Re-run does not make sense on timeouts or when killed by user.
- if (context.hasWorkflowTimedOut() || context.isWorkflowKilledManually()) {
+ // Re-run does not make sense when killed by user.
+ if (context.isWorkflowKilledManually()) {
return;
+ } else if (context.hasWorkflowTimedOut()) {
+ Entity entity = EntityUtil.getEntity(context.getEntityType(), context.getEntityName());
+ Retry retry = getRetry(entity);
+ if (!retry.isOnTimeout()) {
+ return;
+ }
}
handleRerun(context.getClusterName(), context.getEntityType(),
context.getEntityName(), context.getNominalTimeAsISO8601(),
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/src/conf/runtime.properties
----------------------------------------------------------------------
diff --git a/src/conf/runtime.properties b/src/conf/runtime.properties
index f535b0f..2fb148b 100644
--- a/src/conf/runtime.properties
+++ b/src/conf/runtime.properties
@@ -54,6 +54,12 @@ falcon.current.colo=local
# all entities on the Falcon server.
*.falcon.jms.notification.enabled=true
+# Default configs to handle retry for instances(process and feed).
+*.feed.retry.attempts=3
+*.feed.retry.frequency=minutes(5)
+*.feed.retry.policy=exp-backoff
+*.feed.retry.onTimeout=false
+
######### Proxyuser Configuration Start #########
#List of hosts the '#USER#' user is allowed to perform 'doAs 'operations from. The '#USER#' must be replaced with the
http://git-wip-us.apache.org/repos/asf/falcon/blob/8dfc2c93/webapp/src/test/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties
index bd77a1c..09930b8 100644
--- a/webapp/src/test/resources/hdfs-replication.properties
+++ b/webapp/src/test/resources/hdfs-replication.properties
@@ -30,6 +30,7 @@ falcon.recipe.process.frequency=minutes(5)
falcon.recipe.retry.policy=periodic
falcon.recipe.retry.delay=minutes(30)
falcon.recipe.retry.attempts=3
+falcon.recipe.retry.onTimeout=false
drSourceDir=/tmp/test1
drSourceClusterFS=jail://global:00
[2/2] falcon git commit: FALCON-1675 Lifecycle stage is not honoured
when only cluster level lifecycle is implemented (by Ajay Yadava)
Posted by pa...@apache.org.
FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (by Ajay Yadava)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/583651c0
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/583651c0
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/583651c0
Branch: refs/heads/master
Commit: 583651c00d1425b1156b2da92fe8f95148821a84
Parents: 8dfc2c9
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Thu Jan 14 15:28:32 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Jan 14 15:28:32 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../falcon/oozie/feed/FeedBundleBuilder.java | 13 ++---
.../feed/OozieFeedWorkflowBuilderTest.java | 29 +++++++++-
.../feed/fs-local-retention-lifecycle-feed.xml | 61 ++++++++++++++++++++
4 files changed, 95 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cf97ca2..590f17a 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -112,6 +112,8 @@ Proposed Release Version: 0.9
OPTIMIZATIONS
BUG FIXES
+ FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (Ajay Yadava via Pallavi Rao)
+
FALCON-1730 Upgrade enunciate for JDK 8 compatibility(Deepak Barr via Ajay Yadava)
FALCON-1741 Rerun API behaviour different in case of succeeded instances(Pallavi Rao via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
index 1205d91..c758411 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
@@ -18,22 +18,21 @@
package org.apache.falcon.oozie.feed;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.falcon.FalconException;
import org.apache.falcon.Tag;
import org.apache.falcon.entity.FeedHelper;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.Lifecycle;
import org.apache.falcon.lifecycle.LifecyclePolicy;
import org.apache.falcon.oozie.OozieBundleBuilder;
import org.apache.falcon.oozie.OozieCoordinatorBuilder;
import org.apache.falcon.service.LifecyclePolicyMap;
import org.apache.hadoop.fs.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
/**
* Builds oozie bundle for the feed.
*/
@@ -46,8 +45,8 @@ public class FeedBundleBuilder extends OozieBundleBuilder<Feed> {
protected List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
// if feed has lifecycle defined - then use it to create coordinator and wf else fall back
List<Properties> props = new ArrayList<>();
- Lifecycle lifecycle = this.entity.getLifecycle();
- if (lifecycle != null) {
+ boolean isLifeCycleEnabled = FeedHelper.isLifecycleEnabled(this.entity, cluster.getName());
+ if (isLifeCycleEnabled) {
for (String name : FeedHelper.getPolicies(this.entity, cluster.getName())) {
LifecyclePolicy policy = LifecyclePolicyMap.get().get(name);
if (policy == null) {
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 9388c68..a553ea7 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -93,7 +93,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
private Feed tableFeed;
private Feed fsReplFeed;
private Feed lifecycleRetentionFeed;
- private Feed retentionFeed;
+ private Feed lifecycleLocalRetentionFeed;
private Feed fsReplFeedCounter;
private static final String SRC_CLUSTER_PATH = "/feed/src-cluster.xml";
@@ -102,7 +102,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
private static final String TABLE_FEED = "/feed/table-replication-feed.xml";
private static final String FS_REPLICATION_FEED = "/feed/fs-replication-feed.xml";
private static final String FS_RETENTION_LIFECYCLE_FEED = "/feed/fs-retention-lifecycle-feed.xml";
- private static final String FS_RETENTION_ORIG_FEED = "/feed/fs-retention-feed.xml";
+ private static final String FS_LOCAL_RETENTION_LIFECYCLE_FEED = "/feed/fs-local-retention-lifecycle-feed.xml";
private static final String FS_REPLICATION_FEED_COUNTER = "/feed/fs-replication-feed-counters.xml";
@BeforeClass
@@ -137,7 +137,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
fsReplFeedCounter = (Feed) storeEntity(EntityType.FEED, FS_REPLICATION_FEED_COUNTER);
tableFeed = (Feed) storeEntity(EntityType.FEED, TABLE_FEED);
lifecycleRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_LIFECYCLE_FEED);
- retentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_ORIG_FEED);
+ lifecycleLocalRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_LOCAL_RETENTION_LIFECYCLE_FEED);
}
private Entity storeEntity(EntityType type, String resource) throws Exception {
@@ -198,6 +198,29 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
Assert.assertEquals(wfProps.get("jobPriority"), "LOW");
}
+ @Test
+ public void testLocalOnlyRetentionLifecycle() throws Exception {
+ OozieEntityBuilder builder = OozieEntityBuilder.get(lifecycleLocalRetentionFeed);
+ Path bundlePath = new Path("/projects/falcon/");
+ builder.build(trgCluster, bundlePath);
+
+ BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath);
+ List<COORDINATOR> coords = bundle.getCoordinator();
+ COORDINATORAPP coord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath());
+ assertLibExtensions(coord, "retention");
+ HashMap<String, String> props = getCoordProperties(coord);
+ Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION");
+ Assert.assertEquals(coord.getFrequency(), "${coord:hours(12)}");
+ Assert.assertEquals(coord.getTimezone(), "UTC");
+
+ HashMap<String, String> wfProps = getWorkflowProperties(trgMiniDFS.getFileSystem(), coord);
+ Assert.assertEquals(wfProps.get("feedNames"), lifecycleLocalRetentionFeed.getName());
+ Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name()));
+ Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon");
+ Assert.assertEquals(wfProps.get("queueName"), "local");
+ Assert.assertEquals(wfProps.get("limit"), "hours(4)");
+ Assert.assertEquals(wfProps.get("jobPriority"), "HIGH");
+ }
@Test
public void testRetentionFrequency() throws Exception {
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
new file mode 100644
index 0000000..bdf1e59
--- /dev/null
+++ b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
@@ -0,0 +1,61 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="lifecycle retention feed" name="retention-local-lifecycle-test" xmlns="uri:falcon:feed:0.1">
+ <partitions>
+ <partition name="colo"/>
+ <partition name="eventTime"/>
+ <partition name="impressionHour"/>
+ <partition name="pricingModel"/>
+ </partitions>
+
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="minutes(1)"/>
+
+ <clusters>
+ <cluster partition="${cluster.colo}" type="source" name="corp2">
+ <validity end="2099-01-01T00:00Z" start="2012-10-01T12:00Z"/>
+ <retention action="delete" limit="days(10000)"/>
+ <lifecycle>
+ <retention-stage>
+ <frequency>hours(12)</frequency>
+ <queue>local</queue>
+ <priority>HIGH</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(4)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location path="/data/lifecycle/" type="data"/>
+ <location path="/data/regression/fetlrc/billing/stats" type="stats"/>
+ <location path="/data/regression/fetlrc/billing/metadata" type="meta"/>
+ </locations>
+
+ <ACL permission="0x755" group="group" owner="fetl"/>
+ <schema provider="protobuf" location="/databus/streams_local/click_rr/schema/"/>
+ <properties>
+ <property name="maxMaps" value="33" />
+ <property name="mapBandwidth" value="2" />
+ </properties>
+
+
+</feed>