You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by so...@apache.org on 2016/07/05 22:25:26 UTC
falcon git commit: FALCON-1914 Hive mirroring should work for secure
to unsecure & viceversa. Also cleanup artifacts.
Repository: falcon
Updated Branches:
refs/heads/master 3435e2096 -> bd32b610e
FALCON-1914 Hive mirroring should work for secure to unsecure & viceversa. Also cleanup artifacts.
Author: Sowmya Ramesh <sr...@hortonworks.com>
Reviewers: "Balu Vellanki <ba...@apache.org>"
Closes #205 from sowmyaramesh/FALCON-1914
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/bd32b610
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/bd32b610
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/bd32b610
Branch: refs/heads/master
Commit: bd32b610e617ae05027b45e3a5c7672613afdfdc
Parents: 3435e20
Author: Sowmya Ramesh <so...@apache.org>
Authored: Tue Jul 5 15:25:19 2016 -0700
Committer: Sowmya Ramesh <sr...@hortonworks.com>
Committed: Tue Jul 5 15:25:19 2016 -0700
----------------------------------------------------------------------
.../main/META/hive-mirroring-properties.json | 16 +-
.../META/hive-mirroring-secure-properties.json | 191 -------------------
.../runtime/hive-mirroring-secure-template.xml | 45 -----
.../org/apache/falcon/extensions/Extension.java | 2 +-
.../mirroring/hive/HiveMirroringExtension.java | 64 ++++---
5 files changed, 52 insertions(+), 266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json
----------------------------------------------------------------------
diff --git a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json
index a9f3d1b..e019e68 100644
--- a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json
+++ b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-properties.json
@@ -1,5 +1,5 @@
{
- "shortDescription":"This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster.",
+ "shortDescription": "This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster.",
"properties":[
{
"propertyName":"jobName",
@@ -116,6 +116,12 @@
"example":"/apps/hive/dr"
},
{
+ "propertyName":"sourceHive2KerberosPrincipal",
+ "required":true,
+ "description":"Required on secure clusters. Kerberos principal required to access hive servers ",
+ "example":"hive/_HOST@EXAMPLE.COM"
+ },
+ {
"propertyName":"targetCluster",
"required":true,
"description":"target cluster for hive mirroring",
@@ -134,6 +140,12 @@
"example":"/apps/hive/dr"
},
{
+ "propertyName":"targetHive2KerberosPrincipal",
+ "required":true,
+ "description":"Required on secure clusters. Kerberos principal required to access hive servers ",
+ "example":"hive/_HOST@EXAMPLE.COM"
+ },
+ {
"propertyName":"maxEvents",
"required":false,
"description":"To ceil the max events processed each time the job runs. Set it to max value depending on your bandwidth limit. Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!",
@@ -176,4 +188,4 @@
"example":"user1@gmail.com, user2@gmail.com"
}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json
----------------------------------------------------------------------
diff --git a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json b/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json
deleted file mode 100644
index 8ec03b5..0000000
--- a/addons/extensions/hive-mirroring/src/main/META/hive-mirroring-secure-properties.json
+++ /dev/null
@@ -1,191 +0,0 @@
-{
- "shortDescription": "This extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster in secure environment.",
- "properties":[
- {
- "propertyName":"jobName",
- "required":true,
- "description":"Unique job name",
- "example":"hive-monthly-sales-dr"
- },
- {
- "propertyName":"jobClusterName",
- "required":true,
- "description":"Cluster where job should run",
- "example":"backupCluster"
- },
- {
- "propertyName":"jobValidityStart",
- "required":true,
- "description":"Job validity start time",
- "example":"2016-03-03T00:00Z"
- },
- {
- "propertyName":"jobValidityEnd",
- "required":true,
- "description":"Job validity end time",
- "example":"2018-03-13T00:00Z"
- },
- {
- "propertyName":"jobFrequency",
- "required":true,
- "description":"job frequency. Valid frequency types are minutes, hours, days, months",
- "example":"months(1)"
- },
- {
- "propertyName":"jobTimezone",
- "required":false,
- "description":"Time zone for the job",
- "example":"GMT"
- },
- {
- "propertyName":"jobTags",
- "required":false,
- "description":"list of comma separated tags. Key Value Pairs, separated by comma",
- "example":"consumer=consumer@xyz.com, owner=producer@xyz.com, _department_type=forecasting"
- },
- {
- "propertyName":"jobRetryPolicy",
- "required":false,
- "description":"Job retry policy",
- "example":"periodic"
- },
- {
- "propertyName":"jobRetryDelay",
- "required":false,
- "description":"Job retry delay",
- "example":"minutes(30)"
- },
- {
- "propertyName":"jobRetryAttempts",
- "required":false,
- "description":"Job retry attempts",
- "example":"3"
- },
- {
- "propertyName":"jobRetryOnTimeout",
- "required":false,
- "description":"Job retry on timeout",
- "example":true
- },
- {
- "propertyName":"jobAclOwner",
- "required":false,
- "description":"ACL owner",
- "example":"ambari-qa"
- },
- {
- "propertyName":"jobAclGroup",
- "required":false,
- "description":"ACL group",
- "example":"users"
- },
- {
- "propertyName":"jobAclPermission",
- "required":false,
- "description":"ACL permission",
- "example":"0x755"
- },
- {
- "propertyName":"sourceCluster",
- "required":true,
- "description":"Source cluster for hive mirroring",
- "example":"primaryCluster"
- },
- {
- "propertyName":"sourceHiveServer2Uri",
- "required":true,
- "description":"Hive2 server end point",
- "example":"hive2://localhost:10000"
- },
- {
- "propertyName":"sourceDatabases",
- "required":true,
- "description":"For DB level replication specify multiple comma separated databases to replicate",
- "example":"salesDb"
- },
- {
- "propertyName":"sourceTables",
- "required":false,
- "description":"For table level replication specify multiple comma separated tables to replicate",
- "example":"monthly_sales1, monthly_sales2"
- },
- {
- "propertyName":"sourceStagingPath",
- "required":false,
- "description":"Staging path on source",
- "example":"/apps/hive/dr"
- },
- {
- "propertyName":"sourceHive2KerberosPrincipal",
- "required":true,
- "description":"Required on secure clusters. Kerberos principal required to access hive servers ",
- "example":"hive/_HOST@EXAMPLE.COM"
- },
- {
- "propertyName":"targetCluster",
- "required":true,
- "description":"target cluster for hive mirroring",
- "example":"backupCluster"
- },
- {
- "propertyName":"targetHiveServer2Uri",
- "required":true,
- "description":"Hive2 server end point",
- "example":"hive2://localhost:10000"
- },
- {
- "propertyName":"targetStagingPath",
- "required":false,
- "description":"Staging path on target",
- "example":"/apps/hive/dr"
- },
- {
- "propertyName":"targetHive2KerberosPrincipal",
- "required":true,
- "description":"Required on secure clusters. Kerberos principal required to access hive servers ",
- "example":"hive/_HOST@EXAMPLE.COM"
- },
- {
- "propertyName":"maxEvents",
- "required":false,
- "description":"To ceil the max events processed each time the job runs. Set it to max value depending on your bandwidth limit. Setting it to -1 will process all the events but can hog up the bandwidth. Use it judiciously!",
- "example":"10000"
- },
- {
- "propertyName":"replicationMaxMaps",
- "required":false,
- "description":"Maximum number of mappers to use for hive replication",
- "example":"1"
- },
- {
- "propertyName":"distcpMaxMaps",
- "required":false,
- "description":"Maximum number of mappers for DistCP",
- "example":"1"
- },
- {
- "propertyName":"distcpMapBandwidth",
- "required":false,
- "description":"Bandwidth in MB for each mapper in DistCP",
- "example":"100"
- },
- {
- "propertyName":"tdeEncryptionEnabled",
- "required":false,
- "description":"Set this flag to true if TDE encryption is enabled on source and target. Default value is false",
- "example":"true"
- },
- {
- "propertyName":"jobNotificationType",
- "required":false,
- "description":"Email Notification for Falcon instance completion",
- "example":"email"
- },
- {
- "propertyName":"jobNotificationReceivers",
- "required":false,
- "description":"Comma separated email Id's",
- "example":"user1@gmail.com, user2@gmail.com"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml
----------------------------------------------------------------------
diff --git a/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml b/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml
deleted file mode 100644
index 4497bb4..0000000
--- a/addons/extensions/hive-mirroring/src/main/resources/runtime/hive-mirroring-secure-template.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<process name="##jobName##" xmlns="uri:falcon:process:0.1">
- <clusters>
- <!-- source -->
- <cluster name="##jobClusterName##">
- <validity end="##jobValidityEnd##" start="##jobValidityStart##"/>
- </cluster>
- </clusters>
-
- <tags/>
-
- <parallel>1</parallel>
- <!-- Replication needs to run only once to catch up -->
- <order>LAST_ONLY</order>
- <frequency>##jobFrequency##</frequency>
- <timezone>##jobTimezone##</timezone>
-
- <properties>
- <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
- </properties>
-
- <workflow name="##jobWorkflowName##" engine="##jobWorkflowEngine##"
- path="##jobWorkflowPath##" lib="##jobWorkflowLibPath##"/>
- <retry policy="##jobRetryPolicy##" delay="##jobRetryDelay##" attempts="3"/>
- <notification type="##jobNotificationType##" to="##jobNotificationReceivers##"/>
- <ACL/>
-</process>
http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
index 4ab9f5d..6c83fe8 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java
@@ -86,7 +86,7 @@ public class Extension implements ExtensionBuilder {
Map<String, String> extensionResources = store.getExtensionResources(name);
/* Get the resources */
- String extensionTemplate = getExtensionTemplate(store, extensionResources, resourceName);
+ String extensionTemplate = getExtensionTemplate(store, extensionResources, name);
String wfPath = getWFPath(extensionResources, resourceName);
/* Get Lib path */
http://git-wip-us.apache.org/repos/asf/falcon/blob/bd32b610/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java
index 9222e0a..c3bd7a7 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/mirroring/hive/HiveMirroringExtension.java
@@ -139,13 +139,18 @@ public class HiveMirroringExtension extends AbstractExtension {
}
additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_RUN_WRITE_EP.getName(),
ClusterHelper.getStorageUrl(jobCluster));
- if (SecurityUtil.isSecurityEnabled()) {
+
+ // Check if job cluster is secure
+ String jobClusterKerberosPrincipal = ClusterHelper.getPropertyValue(jobCluster, SecurityUtil.NN_PRINCIPAL);
+ if (StringUtils.isNotBlank(jobClusterKerberosPrincipal)) {
// Add -secure and update the resource name
String resourceName = getName().toLowerCase() + SECURE_RESOURCE;
additionalProperties.put(ExtensionProperties.RESOURCE_NAME.getName(), resourceName);
- additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_NN_KERBEROS_PRINCIPAL.getName(),
- ClusterHelper.getPropertyValue(jobCluster, SecurityUtil.NN_PRINCIPAL));
+ } else {
+ jobClusterKerberosPrincipal = NOT_APPLICABLE;
}
+ additionalProperties.put(HiveMirroringExtensionProperties.CLUSTER_FOR_JOB_NN_KERBEROS_PRINCIPAL.getName(),
+ jobClusterKerberosPrincipal);
// Properties for src cluster
String srcClusterName = extensionProperties.getProperty(HiveMirroringExtensionProperties.SOURCE_CLUSTER
@@ -165,20 +170,22 @@ public class HiveMirroringExtension extends AbstractExtension {
additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_TABLES.getName(), ALL_TABLES);
}
- if (SecurityUtil.isSecurityEnabled()) {
- String hive2Principal = extensionProperties.getProperty(HiveMirroringExtensionProperties
- .SOURCE_HIVE2_KERBEROS_PRINCIPAL.getName());
- if (StringUtils.isBlank(hive2Principal)) {
- throw new FalconException("Hive server2 kerberos principal for cluster " + srcCluster.getName()
- + "not passed for extension " + jobName);
- }
+ // Check if source cluster is secure
+ String srcClusterKerberosPrincipal = ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.NN_PRINCIPAL);
+ if (StringUtils.isBlank(srcClusterKerberosPrincipal)) {
+ srcClusterKerberosPrincipal = NOT_APPLICABLE;
+ }
- additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName(),
- ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.NN_PRINCIPAL));
- additionalProperties.put(
- HiveMirroringExtensionProperties.SOURCE_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(),
- ClusterHelper.getPropertyValue(srcCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL));
+ String srcHiveMetastorePrincipal = ClusterHelper.getPropertyValue(
+ srcCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL);
+ if (StringUtils.isBlank(srcHiveMetastorePrincipal)) {
+ srcHiveMetastorePrincipal = NOT_APPLICABLE;
}
+ additionalProperties.put(HiveMirroringExtensionProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName(),
+ srcClusterKerberosPrincipal);
+ additionalProperties.put(
+ HiveMirroringExtensionProperties.SOURCE_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(),
+ srcHiveMetastorePrincipal);
// Properties for target cluster
String targetClusterName = extensionProperties.getProperty(HiveMirroringExtensionProperties.TARGET_CLUSTER
@@ -192,20 +199,23 @@ public class HiveMirroringExtension extends AbstractExtension {
additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN.getName(),
ClusterHelper.getStorageUrl(targetCluster));
- if (SecurityUtil.isSecurityEnabled()) {
- String hive2Principal = extensionProperties.getProperty(HiveMirroringExtensionProperties
- .TARGET_HIVE2_KERBEROS_PRINCIPAL.getName());
- if (StringUtils.isBlank(hive2Principal)) {
- throw new FalconException("Hive server2 kerberos principal for cluster " + targetCluster.getName()
- + "not passed for extension " + jobName);
- }
+ // Check if target cluster is secure
+ String targetClusterKerberosPrincipal = ClusterHelper.getPropertyValue(
+ targetCluster, SecurityUtil.NN_PRINCIPAL);
+ if (StringUtils.isBlank(targetClusterKerberosPrincipal)) {
+ targetClusterKerberosPrincipal = NOT_APPLICABLE;
+ }
- additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName(),
- ClusterHelper.getPropertyValue(targetCluster, SecurityUtil.NN_PRINCIPAL));
- additionalProperties.put(
- HiveMirroringExtensionProperties.TARGET_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(),
- ClusterHelper.getPropertyValue(targetCluster, SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL));
+ String targetHiveMetastorePrincipal = ClusterHelper.getPropertyValue(targetCluster,
+ SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL);
+ if (StringUtils.isBlank(targetHiveMetastorePrincipal)) {
+ targetHiveMetastorePrincipal = NOT_APPLICABLE;
}
+ additionalProperties.put(HiveMirroringExtensionProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName(),
+ targetClusterKerberosPrincipal);
+ additionalProperties.put(
+ HiveMirroringExtensionProperties.TARGET_HIVE_METASTORE_KERBEROS_PRINCIPAL.getName(),
+ targetHiveMetastorePrincipal);
// Misc properties
// Add default properties if not passed