You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ra...@apache.org on 2014/09/12 00:19:31 UTC
[26/41] git commit: FALCON-338 late data recording is enabled by
default for all feeds irrespective of late arrival config. Contributed by
Ajay Yadav
FALCON-338 late data recording is enabled by default for all feeds irrespective of late arrival config. Contributed by Ajay Yadav
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/6f770a34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/6f770a34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/6f770a34
Branch: refs/heads/FALCON-585
Commit: 6f770a34df400f3399300ea371a46c1ae0878b06
Parents: d9c115e
Author: Suhas V <su...@inmobi.com>
Authored: Fri Sep 5 00:24:18 2014 +0530
Committer: Suhas V <su...@inmobi.com>
Committed: Fri Sep 5 00:24:18 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../java/org/apache/falcon/entity/EntityUtil.java | 5 +++++
.../java/org/apache/falcon/entity/EntityUtilTest.java | 14 ++++++++++++++
.../falcon/rerun/handler/TestLateRerunHandler.java | 13 +++++++++++++
4 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2086f5b..3e9dcf6 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -77,6 +77,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-338 - late data recording is enabled by default for all feeds
+ irrespective of late arrival config (Ajay Yadav via Suhas Vasu)
+
FALCON-652 EntityUtils tests are failing (Ajay Yadav via Venkatesh Seetharam)
FALCON-650 Instance list APIs occassionally fail when orderBy set to
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
index 8f258fb..9b66aed 100644
--- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
+++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
@@ -613,6 +613,11 @@ public final class EntityUtil {
return null;
}
+ //If late Arrival is not configured do not process further
+ if(((Feed) entity).getLateArrival() == null){
+ return null;
+ }
+
LateProcess lateProcess = new LateProcess();
lateProcess.setDelay(new Frequency(RuntimeProperties.get().getProperty("feed.late.frequency", "hours(3)")));
lateProcess.setPolicy(
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
index 50ee649..3d501a7 100644
--- a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
@@ -19,10 +19,12 @@
package org.apache.falcon.entity;
import org.apache.falcon.Pair;
+import org.apache.falcon.FalconException;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.feed.LateArrival;
import org.apache.falcon.entity.v0.process.Cluster;
import org.apache.falcon.entity.v0.process.Process;
import org.testng.Assert;
@@ -231,4 +233,16 @@ public class EntityUtilTest extends AbstractTestBase {
Assert.assertEquals(startEndDates.first, expectedStartDate);
Assert.assertEquals(startEndDates.second, expectedEndDate);
}
+
+ @Test
+ public void testGetLateProcessFeed() throws FalconException{
+ Feed feed = new Feed();
+
+ Assert.assertNull(EntityUtil.getLateProcess(feed));
+ LateArrival lateArrival = new LateArrival();
+ lateArrival.setCutOff(Frequency.fromString("days(1)"));
+ feed.setLateArrival(lateArrival);
+ Assert.assertNotNull(EntityUtil.getLateProcess(feed));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
----------------------------------------------------------------------
diff --git a/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java b/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
index 8137f60..93fac13 100644
--- a/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
+++ b/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
@@ -47,4 +47,17 @@ public class TestLateRerunHandler {
cutOff = LateRerunHandler.getCutOffTime(feed, nm);
Assert.assertTrue(cutOff.after(new Date()));
}
+
+ /**
+ * This test checks that LateData Handler is invoked only if LateArrival
+ * is configured for a feed.
+ */
+ @Test
+ public void testFeedLateArrivalCheck() throws FalconException {
+ Feed feed = new Feed();
+ String nominalTime = "2013-10-01T12:00Z";
+ Date cutOff = LateRerunHandler.getCutOffTime(feed, nominalTime);
+ Assert.assertEquals(cutOff, new Date(0));
+
+ }
}