You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/01/14 11:53:01 UTC
[2/2] falcon git commit: FALCON-1675 Lifecycle stage is not honoured
when only cluster level lifecycle is implemented (by Ajay Yadava)
FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (by Ajay Yadava)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/583651c0
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/583651c0
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/583651c0
Branch: refs/heads/master
Commit: 583651c00d1425b1156b2da92fe8f95148821a84
Parents: 8dfc2c9
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Thu Jan 14 15:28:32 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Jan 14 15:28:32 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../falcon/oozie/feed/FeedBundleBuilder.java | 13 ++---
.../feed/OozieFeedWorkflowBuilderTest.java | 29 +++++++++-
.../feed/fs-local-retention-lifecycle-feed.xml | 61 ++++++++++++++++++++
4 files changed, 95 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cf97ca2..590f17a 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -112,6 +112,8 @@ Proposed Release Version: 0.9
OPTIMIZATIONS
BUG FIXES
+ FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (Ajay Yadava via Pallavi Rao)
+
FALCON-1730 Upgrade enunciate for JDK 8 compatibility(Deepak Barr via Ajay Yadava)
FALCON-1741 Rerun API behaviour different in case of succeeded instances(Pallavi Rao via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
index 1205d91..c758411 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
@@ -18,22 +18,21 @@
package org.apache.falcon.oozie.feed;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.falcon.FalconException;
import org.apache.falcon.Tag;
import org.apache.falcon.entity.FeedHelper;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.Lifecycle;
import org.apache.falcon.lifecycle.LifecyclePolicy;
import org.apache.falcon.oozie.OozieBundleBuilder;
import org.apache.falcon.oozie.OozieCoordinatorBuilder;
import org.apache.falcon.service.LifecyclePolicyMap;
import org.apache.hadoop.fs.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
/**
* Builds oozie bundle for the feed.
*/
@@ -46,8 +45,8 @@ public class FeedBundleBuilder extends OozieBundleBuilder<Feed> {
protected List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
// if feed has lifecycle defined - then use it to create coordinator and wf else fall back
List<Properties> props = new ArrayList<>();
- Lifecycle lifecycle = this.entity.getLifecycle();
- if (lifecycle != null) {
+ boolean isLifeCycleEnabled = FeedHelper.isLifecycleEnabled(this.entity, cluster.getName());
+ if (isLifeCycleEnabled) {
for (String name : FeedHelper.getPolicies(this.entity, cluster.getName())) {
LifecyclePolicy policy = LifecyclePolicyMap.get().get(name);
if (policy == null) {
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 9388c68..a553ea7 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -93,7 +93,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
private Feed tableFeed;
private Feed fsReplFeed;
private Feed lifecycleRetentionFeed;
- private Feed retentionFeed;
+ private Feed lifecycleLocalRetentionFeed;
private Feed fsReplFeedCounter;
private static final String SRC_CLUSTER_PATH = "/feed/src-cluster.xml";
@@ -102,7 +102,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
private static final String TABLE_FEED = "/feed/table-replication-feed.xml";
private static final String FS_REPLICATION_FEED = "/feed/fs-replication-feed.xml";
private static final String FS_RETENTION_LIFECYCLE_FEED = "/feed/fs-retention-lifecycle-feed.xml";
- private static final String FS_RETENTION_ORIG_FEED = "/feed/fs-retention-feed.xml";
+ private static final String FS_LOCAL_RETENTION_LIFECYCLE_FEED = "/feed/fs-local-retention-lifecycle-feed.xml";
private static final String FS_REPLICATION_FEED_COUNTER = "/feed/fs-replication-feed-counters.xml";
@BeforeClass
@@ -137,7 +137,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
fsReplFeedCounter = (Feed) storeEntity(EntityType.FEED, FS_REPLICATION_FEED_COUNTER);
tableFeed = (Feed) storeEntity(EntityType.FEED, TABLE_FEED);
lifecycleRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_LIFECYCLE_FEED);
- retentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_ORIG_FEED);
+ lifecycleLocalRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_LOCAL_RETENTION_LIFECYCLE_FEED);
}
private Entity storeEntity(EntityType type, String resource) throws Exception {
@@ -198,6 +198,29 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
Assert.assertEquals(wfProps.get("jobPriority"), "LOW");
}
+ @Test
+ public void testLocalOnlyRetentionLifecycle() throws Exception {
+ OozieEntityBuilder builder = OozieEntityBuilder.get(lifecycleLocalRetentionFeed);
+ Path bundlePath = new Path("/projects/falcon/");
+ builder.build(trgCluster, bundlePath);
+
+ BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath);
+ List<COORDINATOR> coords = bundle.getCoordinator();
+ COORDINATORAPP coord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath());
+ assertLibExtensions(coord, "retention");
+ HashMap<String, String> props = getCoordProperties(coord);
+ Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION");
+ Assert.assertEquals(coord.getFrequency(), "${coord:hours(12)}");
+ Assert.assertEquals(coord.getTimezone(), "UTC");
+
+ HashMap<String, String> wfProps = getWorkflowProperties(trgMiniDFS.getFileSystem(), coord);
+ Assert.assertEquals(wfProps.get("feedNames"), lifecycleLocalRetentionFeed.getName());
+ Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name()));
+ Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon");
+ Assert.assertEquals(wfProps.get("queueName"), "local");
+ Assert.assertEquals(wfProps.get("limit"), "hours(4)");
+ Assert.assertEquals(wfProps.get("jobPriority"), "HIGH");
+ }
@Test
public void testRetentionFrequency() throws Exception {
http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
new file mode 100644
index 0000000..bdf1e59
--- /dev/null
+++ b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
@@ -0,0 +1,61 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<feed description="lifecycle retention feed" name="retention-local-lifecycle-test" xmlns="uri:falcon:feed:0.1">
+ <partitions>
+ <partition name="colo"/>
+ <partition name="eventTime"/>
+ <partition name="impressionHour"/>
+ <partition name="pricingModel"/>
+ </partitions>
+
+ <frequency>minutes(5)</frequency>
+ <timezone>UTC</timezone>
+ <late-arrival cut-off="minutes(1)"/>
+
+ <clusters>
+ <cluster partition="${cluster.colo}" type="source" name="corp2">
+ <validity end="2099-01-01T00:00Z" start="2012-10-01T12:00Z"/>
+ <retention action="delete" limit="days(10000)"/>
+ <lifecycle>
+ <retention-stage>
+ <frequency>hours(12)</frequency>
+ <queue>local</queue>
+ <priority>HIGH</priority>
+ <properties>
+ <property name="retention.policy.agebaseddelete.limit" value="hours(4)"></property>
+ </properties>
+ </retention-stage>
+ </lifecycle>
+ </cluster>
+ </clusters>
+
+ <locations>
+ <location path="/data/lifecycle/" type="data"/>
+ <location path="/data/regression/fetlrc/billing/stats" type="stats"/>
+ <location path="/data/regression/fetlrc/billing/metadata" type="meta"/>
+ </locations>
+
+ <ACL permission="0x755" group="group" owner="fetl"/>
+ <schema provider="protobuf" location="/databus/streams_local/click_rr/schema/"/>
+ <properties>
+ <property name="maxMaps" value="33" />
+ <property name="mapBandwidth" value="2" />
+ </properties>
+
+
+</feed>