You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/04/28 10:28:54 UTC
ambari git commit: AMBARI-10782. HDFS enhanced-configs of keyserver
host/port should be implemented (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk 673980a06 -> 70c66cbd8
AMBARI-10782. HDFS enhanced-configs of keyserver host/port should be implemented (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/70c66cbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/70c66cbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/70c66cbd
Branch: refs/heads/trunk
Commit: 70c66cbd8c54876a795c7a05a1fce15f75f103ca
Parents: 673980a
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Apr 28 01:23:32 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Apr 28 01:23:38 2015 -0700
----------------------------------------------------------------------
.../services/HDFS/configuration/core-site.xml | 15 ++++++
.../services/HDFS/configuration/hadoop-env.xml | 16 ++++++
.../services/HDFS/configuration/hdfs-site.xml | 15 ++++++
.../HDP/2.2/services/HDFS/themes/theme.json | 45 ++++++++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 27 ++++++++++
.../stacks/2.2/common/test_stack_advisor.py | 54 ++++++++++++++++++++
6 files changed, 172 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
index ece7835..c1db0f9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
@@ -28,4 +28,19 @@
<description></description>
</property>
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <value> </value>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_host</name>
+ </property>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_port</name>
+ </property>
+ </depends-on>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
index 77494cd..e4e548f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml
@@ -23,6 +23,22 @@
<configuration>
<!-- hadoop-env.sh -->
<property>
+ <name>keyserver_host</name>
+ <value> </value>
+ <description>Hostnames where Key Management Server is installed</description>
+ <value-attributes>
+ <type>string</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>keyserver_port</name>
+ <value> </value>
+ <description>Port number where Key Management Server is available</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ </property>
+ <property>
<name>content</name>
<description>This is the jinja template for hadoop-env.sh file</description>
<value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-site.xml
index 97d5865..48ef96e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-site.xml
@@ -34,4 +34,19 @@
<description>The path where the JournalNode daemon will store its local state. </description>
</property>
+ <property>
+ <name>dfs.encryption.key.provider.uri</name>
+ <value> </value>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_host</name>
+ </property>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_port</name>
+ </property>
+ </depends-on>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
index 84e61d1..d1dd1f5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
@@ -50,6 +50,31 @@
"column-span": "1"
}
]
+ },
+ {
+ "name": "section-encryption",
+ "display-name": "Encryption",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-columns": "2",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-encryption-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }, {
+ "name": "subsection-encryption-col2",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
}
]
}
@@ -91,6 +116,14 @@
{
"config": "hdfs-site/dfs.datanode.max.transfer.threads",
"subsection-name": "subsection-datanode-col1"
+ },
+ {
+ "config": "hadoop-env/keyserver_host",
+ "subsection-name": "subsection-encryption-col1"
+ },
+ {
+ "config": "hadoop-env/keyserver_port",
+ "subsection-name": "subsection-encryption-col2"
}
]
},
@@ -167,6 +200,18 @@
}
]
}
+ },
+ {
+ "config": "hadoop-env/keyserver_host",
+ "widget": {
+ "type": "string"
+ }
+ },
+ {
+ "config": "hadoop-env/keyserver_port",
+ "widget": {
+ "type": "string"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 3405e8c..a12714f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -173,6 +173,33 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site")
putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'maximum', dataDirsCount)
+ keyserverHostsString = None
+ keyserverPortString = None
+ if "keyserver_host" in services["configurations"]["hadoop-env"]["properties"] and "keyserver_port" in services["configurations"]["hadoop-env"]["properties"]:
+ keyserverHostsString = services["configurations"]["hadoop-env"]["properties"]["keyserver_host"]
+ keyserverPortString = services["configurations"]["hadoop-env"]["properties"]["keyserver_port"]
+
+ if keyserverHostsString is None:
+ # Caller did not specify any value - so we recommend where they are installed
+ rangerKMSServerHosts = self.getHostsWithComponent("RANGER_KMS", "RANGER_KMS_SERVER", services, hosts)
+ if rangerKMSServerHosts is not None and len(rangerKMSServerHosts) > 0:
+ rangerKMSServerHostsArray = []
+ for rangeKMSServerHost in rangerKMSServerHosts:
+ rangerKMSServerHostsArray.append(rangeKMSServerHost["Hosts"]["host_name"])
+ keyserverHostsString = ",".join(rangerKMSServerHostsArray)
+ if "kms-env" in services["configurations"] and "kms_port" in services["configurations"]["kms-env"]["properties"]:
+ keyserverPortString = services["configurations"]["kms-env"]["properties"]["kms_port"]
+
+ if keyserverHostsString is not None and len(keyserverHostsString.strip()) > 0:
+ if keyserverPortString is None or len(keyserverPortString.strip()) < 1:
+ keyserverPortString = ":9292"
+ else:
+ keyserverPortString = ":" + keyserverPortString.strip()
+ putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
+ kmsPath = "kms://http@" + keyserverHostsString.strip() + keyserverPortString + "/kms"
+ putCoreSiteProperty("hadoop.security.key.provider.path", kmsPath)
+ putHdfsSiteProperty("dfs.encryption.key.provider.uri", kmsPath)
+
def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendHiveConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/70c66cbd/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index a86de57..3948b2e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1677,6 +1677,60 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "5000")
self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250")
self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250")
+
+ # Test 4 - KMS empty test from previous call
+ self.assertTrue("dfs.encryption.key.provider.uri" not in configurations["hdfs-site"]["properties"])
+
+ # Test 5 - Calculated from hosts install location
+ services["services"].append(
+ {"StackServices":
+ {"service_name" : "RANGER_KMS",
+ "service_version" : "2.6.0.2.2"
+ },
+ "components":[
+ {
+ "href":"/api/v1/stacks/HDP/versions/2.2/services/RANGER_KMS/components/RANGER_KMS_SERVER",
+ "StackServiceComponents":{
+ "advertise_version":"true",
+ "cardinality":"1+",
+ "component_category":"SLAVE",
+ "component_name":"RANGER_KMS_SERVER",
+ "custom_commands":[
+
+ ],
+ "display_name":"RANGER_KMS_SERVER",
+ "is_client":"false",
+ "is_master":"false",
+ "service_name":"RANGER_KMS",
+ "stack_name":"HDP",
+ "stack_version":"2.2",
+ "hostnames":[
+ "host1"
+ ]
+ },
+ "dependencies":[
+
+ ]
+ }
+ ]
+ })
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
+ self.assertEqual("kms://http@host1:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+
+ # Test 6 - Multiple RANGER_KMS_SERVERs
+ services["services"][len(services["services"])-1]["components"][0]["StackServiceComponents"]["hostnames"].append("host2")
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
+ self.assertEqual("kms://http@host1,host2:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+
+ # Test 6 - Multiple RANGER_KMS_SERVERs and custom port
+ configurations["kms-env"] = {"properties": {"kms_port": "1111"}}
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
+ self.assertEqual("kms://http@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+
+ # Test 7 - Override by API caller
+ configurations["hadoop-env"] = {"properties": {"keyserver_host": "myhost1", "keyserver_port": "2222"}}
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
+ self.assertEqual("kms://http@myhost1:2222/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
def test_validateHDFSConfigurationsEnv(self):
configurations = {}