You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/10/29 06:30:27 UTC
falcon git commit: FALCON-1560 Lifecycle does not allow feed with
frequency greater than days(1). Contributed by Ajay Yadava.
Repository: falcon
Updated Branches:
refs/heads/0.8 6df02522b -> 94b755615
FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1). Contributed by Ajay Yadava.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/94b75561
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/94b75561
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/94b75561
Branch: refs/heads/0.8
Commit: 94b75561571f4d6051a1bf1ad07f7dc140c93039
Parents: 6df0252
Author: Ajay Yadav <aj...@inmobi.com>
Authored: Thu Oct 29 10:30:05 2015 +0530
Committer: Ajay Yadav <aj...@inmobi.com>
Committed: Thu Oct 29 10:30:05 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../java/org/apache/falcon/entity/FeedHelper.java | 10 ++++++----
.../org/apache/falcon/entity/FeedHelperTest.java | 11 +++++++++--
.../falcon/entity/parser/FeedEntityParserTest.java | 16 ++++++++++++++++
4 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/94b75561/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ee2c818..d57a8ec 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -75,6 +75,8 @@ Release version: 0.8
FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao)
BUG FIXES
+ FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1) (Ajay Yadava)
+
FALCON-1556 Falcon build fails when building with hivedr profile(Sowmya Ramesh)
FALCON-1558 Sla API shows pending instance although _SUCCESS directory is present(Pragya Mittal via Ajay Yadava).
http://git-wip-us.apache.org/repos/asf/falcon/blob/94b75561/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
index 5c252a8..f1a22e9 100644
--- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
@@ -46,6 +46,7 @@ import org.apache.falcon.resource.EntityList;
import org.apache.falcon.resource.FeedInstanceResult;
import org.apache.falcon.resource.SchedulableEntityInstance;
import org.apache.falcon.util.BuildProperties;
+import org.apache.falcon.util.DateUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
@@ -804,11 +805,12 @@ public final class FeedHelper {
if (retentionStage != null && retentionStage.getFrequency() != null) {
retentionFrequency = retentionStage.getFrequency();
} else {
- Frequency.TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
- if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes) {
- retentionFrequency = new Frequency("hours(6)");
+ Frequency feedFrequency = feed.getFrequency();
+ Frequency defaultFrequency = new Frequency("hours(6)");
+ if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) {
+ retentionFrequency = defaultFrequency;
} else {
- retentionFrequency = new Frequency("days(1)");
+ retentionFrequency = new Frequency(feedFrequency.toString());
}
}
return retentionFrequency;
http://git-wip-us.apache.org/repos/asf/falcon/blob/94b75561/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
index 4020d36..cd678d6 100644
--- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
@@ -822,15 +822,22 @@ public class FeedHelperTest extends AbstractTestBase {
cluster.setName("cluster1");
clusters.getClusters().add(cluster);
feed.setClusters(clusters);
- Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(1)"));
+ Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(10)"));
feed.setFrequency(new Frequency("hours(1)"));
Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(6)"));
- feed.setFrequency(new Frequency("minutes(5)"));
+ feed.setFrequency(new Frequency("minutes(10)"));
Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(6)"));
+ feed.setFrequency(new Frequency("hours(7)"));
+ Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(7)"));
+
+ feed.setFrequency(new Frequency("days(2)"));
+ Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(2)"));
+
// lifecycle at both level - retention only at global level.
+ feed.setFrequency(new Frequency("hours(1)"));
Lifecycle globalLifecycle = new Lifecycle();
RetentionStage globalRetentionStage = new RetentionStage();
globalRetentionStage.setFrequency(new Frequency("hours(2)"));
http://git-wip-us.apache.org/repos/asf/falcon/blob/94b75561/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 905be68..faa0a41 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -198,6 +198,22 @@ public class FeedEntityParserTest extends AbstractTestBase {
parser.validate(feed); // no validation exception should be thrown
}
+ @Test
+ public void testDefaultRetentionFrequencyConflict() throws Exception {
+ Feed feed = parser.parseAndValidate(this.getClass().getResourceAsStream(FEED3_XML));
+ feed.getLifecycle().getRetentionStage().setFrequency(null);
+ feed.getClusters().getClusters().get(0).getLifecycle().getRetentionStage().setFrequency(null);
+ feed.setFrequency(Frequency.fromString("minutes(10)"));
+ parser.validate(feed); // shouldn't throw a validation exception
+
+
+ feed.setFrequency(Frequency.fromString("hours(7)"));
+ parser.validate(feed); // shouldn't throw a validation exception
+
+ feed.setFrequency(Frequency.fromString("days(2)"));
+ parser.validate(feed); // shouldn't throw a validation exception
+ }
+
@Test(expectedExceptions = ValidationException.class,
expectedExceptionsMessageRegExp = ".*Retention can not be more frequent than data availability.*")
public void testRetentionFrequentThanFeed() throws Exception {