You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/09/22 16:45:42 UTC
falcon git commit: FALCON-1417 Make validity end date optional for
feed / process. Contributed by Pragya Mittal.
Repository: falcon
Updated Branches:
refs/heads/master 55bcc14af -> 75079c2e5
FALCON-1417 Make validity end date optional for feed / process. Contributed by Pragya Mittal.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/75079c2e
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/75079c2e
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/75079c2e
Branch: refs/heads/master
Commit: 75079c2e5166ef517e500109f0015fff6df47b90
Parents: 55bcc14
Author: Ajay Yadava <aj...@gmail.com>
Authored: Tue Sep 22 19:52:49 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Tue Sep 22 19:52:49 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../entity/parser/CrossEntityValidations.java | 7 +++++++
.../falcon/entity/parser/FeedEntityParser.java | 7 +++++++
.../entity/parser/ProcessEntityParser.java | 7 +++++++
.../java/org/apache/falcon/util/DateUtil.java | 4 ++++
.../entity/parser/FeedEntityParserTest.java | 8 ++++++++
.../entity/parser/ProcessEntityParserTest.java | 19 +++++++++++++++++++
7 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 445b56f..6836690 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ Trunk (Unreleased)
FALCON-1027 Falcon proxy user support(Sowmya Ramesh)
IMPROVEMENTS
+ FALCON-1417 Make validity end date optional for feed / process(Pragya Mittal via Ajay Yadava)
+
FALCON-1434 Enhance schedule API to accept key-value properties(Pallavi Rao)
FALCON-1426 Add documentation for Falcon Email notification (Peeyush Bishnoi via Pallavi Rao)
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
index 2696552..18ae754 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
@@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.process.Output;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.entity.v0.process.Validity;
import org.apache.falcon.expression.ExpressionHelper;
+import org.apache.falcon.util.DateUtil;
import java.util.Date;
@@ -45,6 +46,12 @@ public final class CrossEntityValidations {
String clusterName = cluster.getName();
org.apache.falcon.entity.v0.feed.Validity feedValidity = FeedHelper.getCluster(feed,
clusterName).getValidity();
+
+ // Optinal end_date
+ if (feedValidity.getEnd() == null) {
+ feedValidity.setEnd(DateUtil.NEVER);
+ }
+
Date feedStart = feedValidity.getStart();
Date feedEnd = feedValidity.getEnd();
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index 992fc51..4f5599e 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -47,6 +47,7 @@ import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.expression.ExpressionHelper;
import org.apache.falcon.group.FeedGroup;
import org.apache.falcon.group.FeedGroupMap;
+import org.apache.falcon.util.DateUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.slf4j.Logger;
@@ -82,6 +83,12 @@ public class FeedEntityParser extends EntityParser<Feed> {
validateACL(feed);
for (Cluster cluster : feed.getClusters().getClusters()) {
validateEntityExists(EntityType.CLUSTER, cluster.getName());
+
+ // Optinal end_date
+ if (cluster.getValidity().getEnd() == null) {
+ cluster.getValidity().setEnd(DateUtil.NEVER);
+ }
+
validateClusterValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd(),
cluster.getName());
validateClusterHasRegistry(feed, cluster);
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
index 48a4286..10a5265 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java
@@ -40,6 +40,7 @@ import org.apache.falcon.entity.v0.process.Outputs;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.expression.ExpressionHelper;
import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.util.DateUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -79,6 +80,12 @@ public class ProcessEntityParser extends EntityParser<Process> {
+ " is defined more than once for process: " + process.getName());
}
validateEntityExists(EntityType.CLUSTER, clusterName);
+
+ // Optinal end_date
+ if (cluster.getValidity().getEnd() == null) {
+ cluster.getValidity().setEnd(DateUtil.NEVER);
+ }
+
validateProcessValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd());
validateHDFSPaths(process, clusterName);
validateProperties(process);
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/util/DateUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/util/DateUtil.java b/common/src/main/java/org/apache/falcon/util/DateUtil.java
index e736340..f89ef64 100644
--- a/common/src/main/java/org/apache/falcon/util/DateUtil.java
+++ b/common/src/main/java/org/apache/falcon/util/DateUtil.java
@@ -27,6 +27,9 @@ import java.util.TimeZone;
*/
public final class DateUtil {
+ //Friday, April 16, 9999 7:12:55 AM UTC corresponding date
+ public static final Date NEVER = new Date(Long.parseLong("253379862775000"));
+
private DateUtil() {}
public static Date getNextMinute(Date time) throws Exception {
@@ -36,4 +39,5 @@ public final class DateUtil {
insCal.add(Calendar.MINUTE, 1);
return insCal.getTime();
}
+
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 1e9b72f..b6fdb13 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -974,4 +974,12 @@ public class FeedEntityParserTest extends AbstractTestBase {
}
}
+ @Test
+ public void testFeedEndTimeOptional() throws Exception {
+ Feed feed = parser.parseAndValidate(ProcessEntityParserTest.class
+ .getResourceAsStream(FEED_XML));
+ feed.getClusters().getClusters().get(0).getValidity().setEnd(null);
+ parser.validate(feed);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
index a935db3..da22d56 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
@@ -21,6 +21,7 @@ package org.apache.falcon.entity.parser;
import org.apache.falcon.FalconException;
import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.falcon.entity.AbstractTestBase;
+import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
@@ -601,4 +602,22 @@ public class ProcessEntityParserTest extends AbstractTestBase {
}
}
+ @Test
+ public void testProcessEndTimeOptional() throws FalconException {
+ Process process = parser.parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setEnd(null);
+ parser.validate(process);
+ }
+
+ @Test
+ public void testProcessEndTime() throws FalconException {
+ Process process = parser.parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ String feedName = process.getInputs().getInputs().get(0).getFeed();
+ Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName);
+ feedEntity.getClusters().getClusters().get(0).getValidity().setEnd(null);
+ process.getClusters().getClusters().get(0).getValidity().setEnd(null);
+ parser.validate(process);
+ }
}