You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by so...@apache.org on 2015/09/22 20:46:40 UTC
falcon git commit: FALCON-1437 Change DR recipes notification with
Falcon notification. Contributed by Peeyush Bishnoi.
Repository: falcon
Updated Branches:
refs/heads/master 48d0723cc -> 8600e9465
FALCON-1437 Change DR recipes notification with Falcon notification. Contributed by Peeyush Bishnoi.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8600e946
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8600e946
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8600e946
Branch: refs/heads/master
Commit: 8600e94654ea21b5cff1ea363560ffc17078aa53
Parents: 48d0723
Author: Sowmya Ramesh <sr...@hortonworks.com>
Authored: Tue Sep 22 11:46:37 2015 -0700
Committer: Sowmya Ramesh <sr...@hortonworks.com>
Committed: Tue Sep 22 11:46:37 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../resources/hdfs-replication-template.xml | 1 +
.../resources/hdfs-replication-workflow.xml | 42 ----------------
.../main/resources/hdfs-replication.properties | 5 +-
.../hive-disaster-recovery-secure-template.xml | 1 +
.../hive-disaster-recovery-secure-workflow.xml | 50 +-------------------
.../hive-disaster-recovery-secure.properties | 5 +-
.../hive-disaster-recovery-template.xml | 1 +
.../hive-disaster-recovery-workflow.xml | 50 +-------------------
.../resources/hive-disaster-recovery.properties | 5 +-
.../apache/falcon/recipe/RecipeToolOptions.java | 4 +-
.../recipe/util/RecipeProcessBuilderUtils.java | 16 +++++++
.../resources/hdfs-replication-template.xml | 1 +
.../test/resources/hdfs-replication.properties | 5 +-
14 files changed, 41 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 79d7548..5a7c46e 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,8 @@ Trunk (Unreleased)
FALCON-1401 MetadataMappingService fails to add an edge for a process instance(Pallavi Rao)
NEW FEATURES
+ FALCON-1437 Change DR recipes notification with Falcon notification(Peeyush Bishnoi via Sowmya Ramesh)
+
FALCON-1027 Falcon proxy user support(Sowmya Ramesh)
IMPROVEMENTS
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
index de0f748..441a189 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml
@@ -39,5 +39,6 @@
<workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
+ <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
</process>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
index e35773f..942421f 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
@@ -68,48 +68,6 @@
<arg>-availabilityFlag</arg>
<arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg>
</java>
- <ok to="success"/>
- <error to="failure"/>
- </action>
- <decision name="success">
- <switch>
- <case to="successAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="end"/>
- </switch>
- </decision>
- <decision name="failure">
- <switch>
- <case to="failureAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="fail"/>
- </switch>
- </decision>
- <action name="successAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>INFO: HDFS DR workflow ${entityName} completed successfully</subject>
- <body>
- The HDFS DR workflow ${wf:id()} is successful.
- Source = ${drSourceDir}
- Target = ${drTargetClusterFS}${drTargetDir}
- </body>
- </email>
- <ok to="end"/>
- <error to="end"/>
- </action>
- <action name="failureAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>ERROR: HDFS DR workflow ${entityName} failed</subject>
- <body>
- The workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())}
- Source = ${drSourceDir}
- Target = ${drTargetClusterFS}${drTargetDir}
- </body>
- </email>
<ok to="end"/>
<error to="fail"/>
</action>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
index 64ab6b8..7ef4095 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
@@ -73,5 +73,6 @@ distcpMaxMaps=1
# Change it to specify the bandwidth in MB for each mapper in DistCP
distcpMapBandwidth=100
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
index 3afbef0..f0de091 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml
@@ -40,5 +40,6 @@
<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
+ <notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>
</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
index 7362c2e..74902b4 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml
@@ -148,7 +148,7 @@
<arg>lastevents</arg>
</java>
<ok to="export-dr-replication"/>
- <error to="failure"/>
+ <error to="fail"/>
</action>
<!-- Export Replication action -->
<action name="export-dr-replication" cred="hive_src_credentials,hive2_src_credentials">
@@ -246,7 +246,7 @@
<arg>export</arg>
</java>
<ok to="import-dr-replication"/>
- <error to="failure"/>
+ <error to="fail"/>
</action>
<!-- Import Replication action -->
<action name="import-dr-replication" cred="hive_tgt_credentials,hive2_tgt_credentials">
@@ -343,52 +343,6 @@
<arg>-executionStage</arg>
<arg>import</arg>
</java>
- <ok to="success"/>
- <error to="failure"/>
- </action>
- <decision name="success">
- <switch>
- <case to="successAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="end"/>
- </switch>
- </decision>
- <decision name="failure">
- <switch>
- <case to="failureAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="fail"/>
- </switch>
- </decision>
- <action name="successAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject>
- <body>
- The Hive DR workflow ${wf:id()} is successful.
- Source = ${sourceCluster}
- Target = ${targetCluster}
- DB Name = ${sourceDatabase}
- Table Name = ${sourceTable}
- </body>
- </email>
- <ok to="end"/>
- <error to="end"/>
- </action>
- <action name="failureAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>ERROR: Hive DR workflow ${drJobName} failed</subject>
- <body>
- The Hive DR workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())}
- Source = ${sourceCluster}
- Target = ${targetCluster}
- DB Name = ${sourceDatabase}
- Table Name = ${sourceTable}
- </body>
- </email>
<ok to="end"/>
<error to="fail"/>
</action>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
index cc70ac4..df4a8fd 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties
@@ -102,5 +102,6 @@ distcpMaxMaps=1
# Change it to specify the bandwidth in MB for each mapper in DistCP
distcpMapBandwidth=100
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
index 3afbef0..f0de091 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml
@@ -40,5 +40,6 @@
<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
+ <notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>
</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
index c441998..72d40a3 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml
@@ -84,7 +84,7 @@
<arg>lastevents</arg>
</java>
<ok to="export-dr-replication"/>
- <error to="failure"/>
+ <error to="fail"/>
</action>
<!-- Export Replication action -->
<action name="export-dr-replication">
@@ -160,7 +160,7 @@
<arg>export</arg>
</java>
<ok to="import-dr-replication"/>
- <error to="failure"/>
+ <error to="fail"/>
</action>
<!-- Import Replication action -->
<action name="import-dr-replication">
@@ -235,52 +235,6 @@
<arg>-executionStage</arg>
<arg>import</arg>
</java>
- <ok to="success"/>
- <error to="failure"/>
- </action>
- <decision name="success">
- <switch>
- <case to="successAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="end"/>
- </switch>
- </decision>
- <decision name="failure">
- <switch>
- <case to="failureAlert">
- ${drNotificationReceivers ne 'NA'}
- </case>
- <default to="fail"/>
- </switch>
- </decision>
- <action name="successAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject>
- <body>
- The Hive DR workflow ${wf:id()} is successful.
- Source = ${sourceCluster}
- Target = ${targetCluster}
- DB Name = ${sourceDatabase}
- Table Name = ${sourceTable}
- </body>
- </email>
- <ok to="end"/>
- <error to="end"/>
- </action>
- <action name="failureAlert">
- <email xmlns="uri:oozie:email-action:0.2">
- <to>${drNotificationReceivers}</to>
- <subject>ERROR: Hive DR workflow ${drJobName} failed</subject>
- <body>
- The Hive DR workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())}
- Source = ${sourceCluster}
- Target = ${targetCluster}
- DB Name = ${sourceDatabase}
- Table Name = ${sourceTable}
- </body>
- </email>
<ok to="end"/>
<error to="fail"/>
</action>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
index c2cbf5d..c2915fb 100644
--- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
+++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties
@@ -92,5 +92,6 @@ distcpMaxMaps=1
# Change it to specify the bandwidth in MB for each mapper in DistCP
distcpMapBandwidth=100
-##### Email on failure
-drNotificationReceivers=NA
+##### Email Notification for Falcon instance completion
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
index 5df9b0a..0ece90b 100644
--- a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
+++ b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java
@@ -42,7 +42,9 @@ public enum RecipeToolOptions {
RECIPE_ACL_OWNER("falcon.recipe.acl.owner", "Recipe acl owner", false),
RECIPE_ACL_GROUP("falcon.recipe.acl.group", "Recipe acl group", false),
RECIPE_ACL_PERMISSION("falcon.recipe.acl.permission", "Recipe acl permission", false),
- RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false);
+ RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false),
+ RECIPE_NOTIFICATION_TYPE("falcon.recipe.notification.type", "Recipe Notification Type", false),
+ RECIPE_NOTIFICATION_ADDRESS("falcon.recipe.notification.receivers", "Recipe Email Notification receivers", false);
private final String name;
private final String description;
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
index 9522816..9477bb4 100644
--- a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
+++ b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java
@@ -26,6 +26,7 @@ import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.entity.v0.process.ACL;
import org.apache.falcon.entity.v0.process.Cluster;
+import org.apache.falcon.entity.v0.process.Notification;
import org.apache.falcon.entity.v0.process.PolicyType;
import org.apache.falcon.entity.v0.process.Property;
import org.apache.falcon.entity.v0.process.Retry;
@@ -111,6 +112,7 @@ public final class RecipeProcessBuilderUtils {
bindWorkflowProperties(process.getWorkflow(), recipeProperties);
bindRetryProperties(process.getRetry(), recipeProperties);
+ bindNotificationProperties(process.getNotification(), recipeProperties);
bindACLProperties(process.getACL(), recipeProperties);
bindTagsProperties(process, recipeProperties);
bindCustomProperties(process.getProperties(), recipeProperties);
@@ -175,6 +177,20 @@ public final class RecipeProcessBuilderUtils {
}
}
+ private static void bindNotificationProperties(final Notification processNotification,
+ final Properties recipeProperties) {
+ processNotification.setType(recipeProperties.getProperty(
+ RecipeToolOptions.RECIPE_NOTIFICATION_TYPE.getName()));
+
+ String notificationAddress = recipeProperties.getProperty(
+ RecipeToolOptions.RECIPE_NOTIFICATION_ADDRESS.getName());
+ if (StringUtils.isNotBlank(notificationAddress)) {
+ processNotification.setTo(notificationAddress);
+ } else {
+ processNotification.setTo("NA");
+ }
+ }
+
private static void bindACLProperties(final ACL acl,
final Properties recipeProperties) {
String aclowner = recipeProperties.getProperty(RecipeToolOptions.RECIPE_ACL_OWNER.getName());
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication-template.xml b/webapp/src/test/resources/hdfs-replication-template.xml
index b93cc0b..c038484 100644
--- a/webapp/src/test/resources/hdfs-replication-template.xml
+++ b/webapp/src/test/resources/hdfs-replication-template.xml
@@ -39,5 +39,6 @@
<workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
+ <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties
index 1b485b4..bd77a1c 100644
--- a/webapp/src/test/resources/hdfs-replication.properties
+++ b/webapp/src/test/resources/hdfs-replication.properties
@@ -41,5 +41,6 @@ distcpMaxMaps=1
# Change it to specify the bandwidth in MB for each mapper in DistCP
distcpMapBandwidth=100
-##### Email on failure
-drNotificationReceivers=NA
\ No newline at end of file
+##### Email Notification for Falcon instance completion ####
+falcon.recipe.notification.type=email
+falcon.recipe.notification.receivers=NA
\ No newline at end of file