You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/06/12 20:19:32 UTC
ambari git commit: AMBARI-11870. HDP-2.3 'hive.tez.java.opts' should
not contain Xmx and add additional JDK1.8 params (srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 9e34a010f -> 5b98e7e93
AMBARI-11870. HDP-2.3 'hive.tez.java.opts' should not contain Xmx and add additional JDK1.8 params (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5b98e7e9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5b98e7e9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5b98e7e9
Branch: refs/heads/branch-2.1
Commit: 5b98e7e93d657247c8d2a640bf312d51a3a1527e
Parents: 9e34a01
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu Jun 11 15:52:15 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Fri Jun 12 11:12:38 2015 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 3 +
.../stacks/HDP/2.3/services/stack_advisor.py | 13 +
.../stacks/2.3/common/test_stack_advisor.py | 272 +++++++++++++++++++
3 files changed, 288 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b98e7e9/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7588b02..7085c4d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -628,6 +628,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return self.getErrorItem("Config's default value can't be null or undefined")
if not checkXmxValueFormat(value):
return self.getErrorItem('Invalid value format')
+ if not checkXmxValueFormat(defaultValue):
+ # if default value does not contain Xmx, then there is no point in validating existing value
+ return None
valueInt = formatXmxSizeToBytes(getXmxSize(value))
defaultValueXmx = getXmxSize(defaultValue)
defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b98e7e9/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 0838e4a..076c202 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -139,12 +139,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
# hive_security_authorization == 'ranger'
if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "ranger":
putHiveServerProperty("hive.security.authorization.manager", "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory")
+ # TEZ JVM options
+ jvmGCParams = "-XX:+UseParallelGC"
+ if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]:
+ # JDK8 needs different parameters
+ match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"])
+ if match and len(match.groups()) > 0:
+ # Is version >= 1.8
+ versionSplits = re.split("\.", match.group(1))
+ if versionSplits and len(versionSplits)>1 and int(versionSplits[0])>0 and int(versionSplits[1])>7:
+ jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
+ putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
+
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b98e7e9/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index b72f0e2..07e3258 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -225,3 +225,275 @@ class TestHDP23StackAdvisor(TestCase):
expected["hbase-env"]["properties"]["hbase_regionserver_heapsize"] = "4096"
self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations, expected)
+
+
+ def test_recommendHiveConfigurations(self):
+ self.maxDiff = None
+ configurations = {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "256",
+ "yarn.scheduler.maximum-allocation-mb": "8192",
+ },
+ },
+ "capacity-scheduler": {
+ "properties": {
+ "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+ }
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ 'capacity-scheduler': {
+ 'properties': {
+ 'yarn.scheduler.capacity.root.queues': 'queue1,queue2'
+ }
+ },
+ 'yarn-site': {
+ 'properties': {
+ 'yarn.scheduler.minimum-allocation-mb': '256',
+ 'yarn.scheduler.maximum-allocation-mb': '8192'
+ }
+ },
+ 'hive-env': {
+ 'properties': {
+ 'cost_based_optimizer': 'On',
+ 'hive_exec_orc_storage_strategy': 'SPEED',
+ 'hive_security_authorization': 'None',
+ 'hive_timeline_logging_enabled': 'true',
+ 'hive_txn_acid': 'off'
+ }
+ },
+ 'hive-site': {
+ 'properties': {
+ 'hive.server2.enable.doAs': 'true',
+ 'hive.server2.tez.default.queues': "queue1,queue2",
+ 'hive.server2.tez.initialize.default.sessions': 'false',
+ 'hive.server2.tez.sessions.per.default.queue': '1',
+ 'hive.auto.convert.join.noconditionaltask.size': '268435456',
+ 'hive.cbo.enable': 'true',
+ 'hive.compactor.initiator.on': 'false',
+ 'hive.compactor.worker.threads': '0',
+ 'hive.compute.query.using.stats': 'true',
+ 'hive.enforce.bucketing': 'false',
+ 'hive.exec.dynamic.partition.mode': 'strict',
+ 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.orc.compression.strategy': 'SPEED',
+ 'hive.exec.orc.default.compress': 'ZLIB',
+ 'hive.exec.orc.default.stripe.size': '67108864',
+ 'hive.exec.orc.encoding.strategy': 'SPEED',
+ 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.reducers.bytes.per.reducer': '67108864',
+ 'hive.execution.engine': 'mr',
+ 'hive.optimize.index.filter': 'true',
+ 'hive.optimize.sort.dynamic.partition': 'false',
+ 'hive.prewarm.enabled': 'false',
+ 'hive.prewarm.numcontainers': '3',
+ 'hive.security.authorization.enabled': 'false',
+ 'hive.server2.use.SSL': 'false',
+ 'hive.stats.fetch.column.stats': 'true',
+ 'hive.stats.fetch.partition.stats': 'true',
+ 'hive.support.concurrency': 'false',
+ 'hive.tez.auto.reducer.parallelism': 'true',
+ 'hive.tez.container.size': '768',
+ 'hive.tez.dynamic.partition.pruning': 'true',
+ 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
+ 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager',
+ 'hive.vectorized.execution.enabled': 'true',
+ 'hive.vectorized.execution.reduce.enabled': 'false',
+ 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
+ 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
+ },
+ 'property_attributes': {
+ 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+ 'hive.server2.authentication.pam.services': {'delete': 'true'},
+ 'hive.server2.custom.authentication.class': {'delete': 'true'},
+ 'hive.server2.authentication.ldap.baseDN': {'delete': 'true'},
+ 'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
+ 'hive.server2.authentication.kerberos.keytab': {'delete': 'true'},
+ 'hive.server2.authentication.ldap.url': {'delete': 'true'},
+ 'hive.server2.tez.default.queues': {
+ 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}]
+ }
+ }
+ },
+ 'hiveserver2-site': {
+ 'properties': {
+ },
+ 'property_attributes': {
+ 'hive.security.authorization.manager': {'delete': 'true'},
+ 'hive.security.authorization.enabled': {'delete': 'true'},
+ 'hive.security.authenticator.manager': {'delete': 'true'}
+ }
+ }
+ }
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+ "StackServices": {
+ "service_name": "YARN",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "APP_TIMELINE_SERVER",
+ "display_name": "App Timeline Server",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1-2",
+ "component_category": "MASTER",
+ "component_name": "RESOURCEMANAGER",
+ "display_name": "ResourceManager",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "CLIENT",
+ "component_name": "YARN_CLIENT",
+ "display_name": "YARN Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": configurations,
+ "changed-configurations": [ ]
+
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6402.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6403.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.7
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.8
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.8_44'}
+ expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.8
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'}
+ expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ def test_validateHiveConfigurations(self):
+ properties = {"hive_security_authorization": "None",
+ "hive.exec.orc.default.stripe.size": "8388608",
+ 'hive.tez.container.size': '2048',
+ 'hive.tez.java.opts': '-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
+ 'hive.auto.convert.join.noconditionaltask.size': '1100000000'}
+ recommendedDefaults = {'hive.tez.container.size': '1024',
+ 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
+ 'hive.auto.convert.join.noconditionaltask.size': '1000000000'}
+ configurations = {
+ "hive-site": {
+ "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'}
+ },
+ "hive-env": {
+ "properties": {"hive_security_authorization": "None"}
+ }
+ }
+ services = {
+ "services": []
+ }
+
+ # Test for 'ranger-hive-plugin-properties' not being in configs
+ res_expected = []
+ res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, services, {})
+ self.assertEquals(res, res_expected)
+