You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/10/01 12:44:10 UTC
falcon git commit: FALCON-1499 Lifecycle Retention Issues.
Contributed by Ajay Yadava.
Repository: falcon
Updated Branches:
refs/heads/master 8e0d5c584 -> 4783a19c4
FALCON-1499 Lifecycle Retention Issues. Contributed by Ajay Yadava.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4783a19c
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4783a19c
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4783a19c
Branch: refs/heads/master
Commit: 4783a19c4cfbb673a647b4382961fb0df9b034a6
Parents: 8e0d5c5
Author: Ajay Yadava <aj...@gmail.com>
Authored: Thu Oct 1 12:28:34 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Thu Oct 1 13:48:55 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
common/src/main/resources/startup.properties | 4 +++-
.../engine/oozie/retention/AgeBasedCoordinatorBuilder.java | 2 ++
.../engine/oozie/retention/AgeBasedWorkflowBuilder.java | 9 +++++++--
src/conf/startup.properties | 4 +++-
5 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 22aa5ff..b2104f4 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -47,6 +47,8 @@ Trunk (Unreleased)
FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao)
BUG FIXES
+ FALCON-1499 Lifecycle Retention Issues(Ajay Yadava)
+
FALCON-1469 Flaky EntityManagerJerseyIT#testDuplicateSubmitCommands(Sandeep Samudrala via Ajay Yadava)
FALCON-1470 HiveDRStatusStoreTest should fail when using fakeGroup to create StatusStore(Balu Vellanki via Sowmya Ramesh)
http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 3383129..e853c5a 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -34,6 +34,8 @@
*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.service.FeedSLAMonitoringService,\
+ org.apache.falcon.service.LifecyclePolicyMap,\
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
@@ -76,7 +78,7 @@
#Configurations used in UTs
debug.config.store.uri=file://${user.dir}/target/store
#Location to store state of Feed SLA monitoring service
-debug.pending.feed.instances.store.uri = file://${user.dir}/data/sla/pendingfeedinstances
+debug.feed.sla.service.store.uri= file://${user.dir}/target/data/sla/pendingfeedinstances
debug.config.oozie.conf.uri=${user.dir}/target/oozie
debug.system.lib.location=${system.lib.location}
debug.broker.url=vm://localhost
http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
index 4601070..e3e7fec 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
@@ -83,6 +83,8 @@ public final class AgeBasedCoordinatorBuilder {
CONTROLS controls = new CONTROLS();
controls.setExecution(ExecutionType.LAST_ONLY.value());
controls.setTimeout(String.valueOf(frequencyInMillis / (1000 * 60)));
+ controls.setConcurrency("1");
+ controls.setThrottle("1");
coord.setControls(controls);
coord.setFrequency("${coord:" + retentionFrequency.toString() + "}");
http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
index 0a87213..34b186e 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
@@ -18,6 +18,7 @@
package org.apache.falcon.lifecycle.engine.oozie.retention;
+import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.LifeCycle;
import org.apache.falcon.Tag;
@@ -82,8 +83,12 @@ public final class AgeBasedWorkflowBuilder {
props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.EVICTION));
// override the queueName and priority
RetentionStage retentionStage = FeedHelper.getRetentionStage(feed, cluster.getName());
- props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue());
- props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority());
+ if (StringUtils.isNotBlank(retentionStage.getQueue())) {
+ props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue());
+ }
+ if (StringUtils.isNotBlank(retentionStage.getPriority())) {
+ props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority());
+ }
if (EntityUtil.isTableStorageType(cluster, feed)) {
setupHiveCredentials(cluster, buildPath, workflow);
http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index 8e4ce97..37cb044 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -39,6 +39,7 @@
org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
org.apache.falcon.service.ProcessSubscriberService,\
org.apache.falcon.service.FeedSLAMonitoringService,\
+ org.apache.falcon.service.LifecyclePolicyMap,\
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
@@ -48,7 +49,8 @@
org.apache.falcon.service.ProxyUserService
##### Prism Services #####
-prism.application.services=org.apache.falcon.entity.store.ConfigurationStore
+prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\
+ org.apache.falcon.entity.store.ConfigurationStore
# List of Lifecycle policies configured.